{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import multiprocessing as mp\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.autograd import Variable\n",
    "from collections import namedtuple\n",
    "\n",
    "from PIL import Image\n",
    "import os\n",
    "import os.path\n",
    "import errno\n",
    "import codecs\n",
    "import copy\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starts\n"
     ]
    }
   ],
   "source": [
    "print('starts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.cuda.device_count() 4\n",
      "torch.cuda.current_device() 2\n",
      "torch.cuda.current_device() 2\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "np.random.seed(0)\n",
    "print(\"torch.cuda.device_count()\", torch.cuda.device_count())\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())\n",
    "torch.cuda.set_device(2)\n",
    "print(\"torch.cuda.current_device()\", torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_ranks(x):\n",
    "  \"\"\"\n",
    "  Returns ranks in [0, len(x))\n",
    "  Note: This is different from scipy.stats.rankdata, which returns ranks in [1, len(x)].\n",
    "  (https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py)\n",
    "  \"\"\"\n",
    "  assert x.ndim == 1\n",
    "  ranks = np.empty(len(x), dtype=int)\n",
    "  ranks[x.argsort()] = np.arange(len(x))\n",
    "  return ranks\n",
    "\n",
    "def compute_centered_ranks(x):\n",
    "  \"\"\"\n",
    "  https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py\n",
    "  \"\"\"\n",
    "  y = compute_ranks(x.ravel()).reshape(x.shape).astype(np.float32)\n",
    "  y /= (x.size - 1)\n",
    "  y -= .5\n",
    "  return y\n",
    "\n",
    "def compute_weight_decay(weight_decay, model_param_list):\n",
    "  model_param_grid = np.array(model_param_list)\n",
    "  return - weight_decay * np.mean(model_param_grid * model_param_grid, axis=1)\n",
    "\n",
    "class CMAES:\n",
    "  '''CMA-ES wrapper.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               popsize=255,           # population size\n",
    "               done_threshold=1e-6,   # threshold when we say we are done\n",
    "               weight_decay=0.01):    # weight decay coefficient\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.done_threshold = done_threshold\n",
    "    self.popsize = popsize\n",
    "    self.weight_decay = weight_decay\n",
    "    self.solutions = None\n",
    "\n",
    "    import cma\n",
    "    self.es = cma.CMAEvolutionStrategy( self.num_params * [0],\n",
    "                                        self.sigma_init,\n",
    "                                        {'popsize': self.popsize,\n",
    "                                         'ftarget': self.done_threshold,\n",
    "                                        })\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.es.result[6]\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    self.solutions = np.array(self.es.ask())\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    reward_table = -np.array(reward_table_result)\n",
    "    if self.weight_decay > 0:\n",
    "      l2_decay = compute_weight_decay(self.weight_decay, self.solutions)\n",
    "      reward_table += l2_decay\n",
    "    self.es.tell(self.solutions, (reward_table).tolist()) # convert minimizer to maximizer.\n",
    "\n",
    "  def done(self):\n",
    "    return self.es.stop()\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.es.result[5] # mean solution, presumably better with noise\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.es.result[0] # best evaluated solution\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    r = self.es.result\n",
    "    return (r[0], -r[1], -r[1], r[6])\n",
    "\n",
    "class SimpleES:\n",
    "  '''Simple Evolution Strategies.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.10,       # initial standard deviation\n",
    "               sigma_alpha=0.20,      # learning rate for standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               done_threshold=1e-6,   # threshold when we say we are done\n",
    "               average_baseline=True, # set baseline to average of batch\n",
    "               weight_decay=0.01,     # weight decay coefficient\n",
    "               rank_fitness=True,     # use rank rather than fitness numbers\n",
    "               forget_best=True):     # don't keep the historical best solution\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_alpha = sigma_alpha\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "    self.average_baseline = average_baseline\n",
    "    if self.average_baseline:\n",
    "      assert (self.popsize % 2 == 0), \"Population size must be even\"\n",
    "      self.batch_size = int(self.popsize / 2)\n",
    "    else:\n",
    "      assert (self.popsize & 1), \"Population size must be odd\"\n",
    "      self.batch_size = int((self.popsize - 1) / 2)\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "    self.forget_best = forget_best\n",
    "    self.batch_reward = np.zeros(self.batch_size * 2)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.sigma = np.ones(self.num_params) * self.sigma_init\n",
    "    self.curr_best_mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.weight_decay = weight_decay\n",
    "    self.rank_fitness = rank_fitness\n",
    "    if self.rank_fitness:\n",
    "      self.forget_best = True # always forget the best one if we rank\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.batch_size, self.num_params) * self.sigma.reshape(1, self.num_params)\n",
    "    self.epsilon_full = np.concatenate([self.epsilon, - self.epsilon])\n",
    "    if self.average_baseline:\n",
    "      epsilon = self.epsilon_full\n",
    "    else:\n",
    "      # first population is mu, then positive epsilon, then negative epsilon\n",
    "      epsilon = np.concatenate([np.zeros((1, self.num_params)), self.epsilon_full])\n",
    "    solutions = self.mu.reshape(1, self.num_params) + epsilon\n",
    "    self.solutions = solutions\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    reward_table = np.array(reward_table_result)\n",
    "    \n",
    "    if self.rank_fitness:\n",
    "      reward_table = compute_centered_ranks(reward_table)\n",
    "    \n",
    "    if self.weight_decay > 0:\n",
    "      l2_decay = compute_weight_decay(self.weight_decay, self.solutions)\n",
    "      reward_table += l2_decay\n",
    "\n",
    "    reward_offset = 1\n",
    "    if self.average_baseline:\n",
    "      b = np.mean(reward_table)\n",
    "      reward_offset = 0\n",
    "    else:\n",
    "      b = reward_table[0] # baseline\n",
    "      \n",
    "    reward = reward_table[reward_offset:]\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    if (best_reward > b or self.average_baseline):\n",
    "      best_mu = self.mu + self.epsilon_full[idx[0]]\n",
    "      best_reward = reward[idx[0]]\n",
    "    else:\n",
    "      best_mu = self.mu\n",
    "      best_reward = b\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # adaptive sigma\n",
    "    # normalization\n",
    "    stdev_reward = reward.std()\n",
    "    epsilon = self.epsilon\n",
    "    sigma = self.sigma\n",
    "    S = ((epsilon * epsilon - (sigma * sigma).reshape(1, self.num_params)) / sigma.reshape(1, self.num_params))\n",
    "    reward_avg = (reward[:self.batch_size] + reward[self.batch_size:]) / 2.0\n",
    "    rS = reward_avg - b\n",
    "    delta_sigma = (np.dot(rS, S)) / (2 * self.batch_size * stdev_reward)\n",
    "\n",
    "    # move mean to the average of the best idx means\n",
    "    self.mu += self.epsilon_full[idx].mean(axis=0)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    change_sigma = self.sigma_alpha * delta_sigma\n",
    "    change_sigma = np.minimum(change_sigma, self.sigma)\n",
    "    change_sigma = np.maximum(change_sigma, - 0.5 * self.sigma)\n",
    "    self.sigma += change_sigma\n",
    "    self.sigma[self.sigma > self.sigma_limit] *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class SimpleGA:\n",
    "  '''Simple Genetic Algorithm.'''\n",
    "  def __init__(self, num_params,      # number of model parameters\n",
    "               sigma_init=0.1,        # initial standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,      # stop annealing if less than this\n",
    "               popsize=255,           # population size\n",
    "               elite_ratio=0.1,       # percentage of the elites\n",
    "               forget_best=False,     # forget the historical best elites\n",
    "               weight_decay=0.01,     # weight decay coefficient\n",
    "              ):\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.popsize = popsize\n",
    "\n",
    "    self.elite_ratio = elite_ratio\n",
    "    self.elite_popsize = int(self.popsize * self.elite_ratio)\n",
    "\n",
    "    self.sigma = self.sigma_init\n",
    "    self.elite_params = np.zeros((self.elite_popsize, self.num_params))\n",
    "    self.elite_rewards = np.zeros(self.elite_popsize)\n",
    "    self.best_param = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_iteration = True\n",
    "    self.forget_best = forget_best\n",
    "    self.weight_decay = weight_decay\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    return self.sigma # same sigma for all parameters.\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    self.epsilon = np.random.randn(self.popsize, self.num_params) * self.sigma\n",
    "    solutions = []\n",
    "    \n",
    "    def mate(a, b):\n",
    "      c = np.copy(a)\n",
    "      idx = np.where(np.random.rand((c.size)) > 0.5)\n",
    "      c[idx] = b[idx]\n",
    "      return c\n",
    "    \n",
    "    elite_range = range(self.elite_popsize)\n",
    "    for i in range(self.popsize):\n",
    "      idx_a = np.random.choice(elite_range)\n",
    "      idx_b = np.random.choice(elite_range)\n",
    "      child_params = mate(self.elite_params[idx_a], self.elite_params[idx_b])\n",
    "      solutions.append(child_params + self.epsilon[i])\n",
    "\n",
    "    solutions = np.array(solutions)\n",
    "    self.solutions = solutions\n",
    "\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    reward_table = np.array(reward_table_result)\n",
    "    \n",
    "    if self.weight_decay > 0:\n",
    "      l2_decay = compute_weight_decay(self.weight_decay, self.solutions)\n",
    "      reward_table += l2_decay\n",
    "\n",
    "    if (not self.forget_best or self.first_iteration):\n",
    "      reward = reward_table\n",
    "      solution = self.solutions\n",
    "    else:\n",
    "      reward = np.concatenate([reward_table, self.elite_rewards])\n",
    "      solution = np.concatenate([self.solutions, self.elite_params])\n",
    "\n",
    "    idx = np.argsort(reward)[::-1][0:self.elite_popsize]\n",
    "\n",
    "    self.elite_rewards = reward[idx]\n",
    "    self.elite_params = solution[idx]\n",
    "\n",
    "    self.curr_best_reward = self.elite_rewards[0]\n",
    "    \n",
    "    if self.first_iteration or (self.curr_best_reward > self.best_reward):\n",
    "      self.first_iteration = False\n",
    "      self.best_reward = self.elite_rewards[0]\n",
    "      self.best_param = np.copy(self.elite_params[0])\n",
    "\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.elite_params[0]\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_param\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_param, self.best_reward, self.curr_best_reward, self.sigma)\n",
    "\n",
    "class OpenES:\n",
    "  ''' Basic Version of OpenAI Evolution Strategies.'''\n",
    "  def __init__(self, num_params,             # number of model parameters\n",
    "               sigma_init=0.1,               # initial standard deviation\n",
    "               sigma_decay=0.999,            # anneal standard deviation\n",
    "               sigma_limit=0.01,             # stop annealing if less than this\n",
    "               learning_rate=0.01,           # learning rate for standard deviation\n",
    "               learning_rate_decay = 0.9999, # annealing the learning rate\n",
    "               learning_rate_limit = 0.001,  # stop annealing learning rate\n",
    "               popsize=255,                  # population size\n",
    "               antithetic=False,             # whether to use antithetic sampling\n",
    "               weight_decay=0.01,            # weight decay coefficient\n",
    "               rank_fitness=True,            # use rank rather than fitness numbers\n",
    "               forget_best=True):            # forget historical best\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma = sigma_init\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.learning_rate = learning_rate\n",
    "    self.learning_rate_decay = learning_rate_decay\n",
    "    self.learning_rate_limit = learning_rate_limit\n",
    "    self.popsize = popsize\n",
    "    self.antithetic = antithetic\n",
    "    if self.antithetic:\n",
    "      assert (self.popsize % 2 == 0), \"Population size must be even\"\n",
    "      self.half_popsize = int(self.popsize / 2)\n",
    "\n",
    "    self.reward = np.zeros(self.popsize)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.forget_best = forget_best\n",
    "    self.weight_decay = weight_decay\n",
    "    self.rank_fitness = rank_fitness\n",
    "    if self.rank_fitness:\n",
    "      self.forget_best = True # always forget the best one if we rank\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    if self.antithetic:\n",
    "      self.epsilon_half = np.random.randn(self.half_popsize, self.num_params)\n",
    "      self.epsilon = np.concatenate([self.epsilon_half, - self.epsilon_half])\n",
    "    else:\n",
    "      self.epsilon = np.random.randn(self.popsize, self.num_params)\n",
    "\n",
    "    self.solutions = self.mu.reshape(1, self.num_params) + self.epsilon * self.sigma\n",
    "\n",
    "    return self.solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "    \n",
    "    reward = np.array(reward_table_result)\n",
    "    \n",
    "    if self.rank_fitness:\n",
    "      reward = compute_centered_ranks(reward)\n",
    "    \n",
    "    if self.weight_decay > 0:\n",
    "      l2_decay = compute_weight_decay(self.weight_decay, self.solutions)\n",
    "      reward += l2_decay\n",
    "\n",
    "    idx = np.argsort(reward)[::-1]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    best_mu = self.solutions[idx[0]]\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # main bit:\n",
    "    # standardize the rewards to have a gaussian distribution\n",
    "    normalized_reward = (reward - np.mean(reward)) / np.std(reward)\n",
    "    self.mu += self.learning_rate/(self.popsize*self.sigma)*np.dot(self.epsilon.T, normalized_reward)\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    if (self.sigma > self.sigma_limit):\n",
    "      self.sigma *= self.sigma_decay\n",
    "\n",
    "    if (self.learning_rate > self.learning_rate_limit):\n",
    "      self.learning_rate *= self.learning_rate_decay\n",
    "\n",
    "  def done(self):\n",
    "    return False\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "\n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PEPG:\n",
    "  '''Extension of PEPG with bells and whistles.'''\n",
    "  def __init__(self, num_params,             # number of model parameters\n",
    "               sigma_init=0.10,              # initial standard deviation\n",
    "               sigma_alpha=0.20,             # learning rate for standard deviation\n",
    "               sigma_decay=0.999,            # anneal standard deviation\n",
    "               sigma_limit=0.01,             # stop annealing if less than this\n",
    "               learning_rate=0.01,           # learning rate for standard deviation\n",
    "               learning_rate_decay = 0.9999, # annealing the learning rate\n",
    "               learning_rate_limit = 0.001,  # stop annealing learning rate\n",
    "               popsize=255,                  # population size\n",
    "               done_threshold=1e-6,          # threshold when we say we are done\n",
    "               average_baseline=True,        # set baseline to average of batch\n",
    "               weight_decay=0.01,            # weight decay coefficient\n",
    "               rank_fitness=True,            # use rank rather than fitness numbers\n",
    "               forget_best=True):            # don't keep the historical best solution\n",
    "\n",
    "    self.num_params = num_params\n",
    "    self.sigma_init = sigma_init\n",
    "    self.sigma_alpha = sigma_alpha\n",
    "    self.sigma_decay = sigma_decay\n",
    "    self.sigma_limit = sigma_limit\n",
    "    self.learning_rate = learning_rate\n",
    "    self.learning_rate_decay = learning_rate_decay\n",
    "    self.learning_rate_limit = learning_rate_limit\n",
    "    self.popsize = popsize\n",
    "    self.average_baseline = average_baseline\n",
    "    if self.average_baseline:\n",
    "      assert (self.popsize % 2 == 0), \"Population size must be even\"\n",
    "      self.batch_size = int(self.popsize / 2)\n",
    "    else:\n",
    "      assert (self.popsize & 1), \"Population size must be odd\"\n",
    "      self.batch_size = int((self.popsize - 1) / 2)\n",
    "    self.forget_best = forget_best\n",
    "    self.batch_reward = np.zeros(self.batch_size * 2)\n",
    "    self.mu = np.zeros(self.num_params)\n",
    "    self.sigma = np.ones(self.num_params) * self.sigma_init\n",
    "    self.curr_best_mu = np.zeros(self.num_params)\n",
    "    self.best_mu = np.zeros(self.num_params)\n",
    "    self.best_reward = 0\n",
    "    self.first_interation = True\n",
    "    self.weight_decay = weight_decay\n",
    "    self.rank_fitness = rank_fitness\n",
    "    if self.rank_fitness:\n",
    "      self.forget_best = True # always forget the best one if we rank\n",
    "    self.done_threshold = done_threshold\n",
    "\n",
    "  def rms_stdev(self):\n",
    "    sigma = self.sigma\n",
    "    return np.mean(np.sqrt(sigma*sigma))\n",
    "\n",
    "  def ask(self):\n",
    "    '''returns a list of parameters'''\n",
    "    # antithetic sampling\n",
    "    self.epsilon = np.random.randn(self.batch_size, self.num_params) * self.sigma.reshape(1, self.num_params)\n",
    "    self.epsilon_full = np.concatenate([self.epsilon, - self.epsilon])\n",
    "    if self.average_baseline:\n",
    "      epsilon = self.epsilon_full\n",
    "    else:\n",
    "      # first population is mu, then positive epsilon, then negative epsilon\n",
    "      epsilon = np.concatenate([np.zeros((1, self.num_params)), self.epsilon_full])\n",
    "    solutions = self.mu.reshape(1, self.num_params) + epsilon\n",
    "    self.solutions = solutions\n",
    "    return solutions\n",
    "\n",
    "  def tell(self, reward_table_result):\n",
    "    # input must be a numpy float array\n",
    "    assert(len(reward_table_result) == self.popsize), \"Inconsistent reward_table size reported.\"\n",
    "\n",
    "    reward_table = np.array(reward_table_result)\n",
    "    \n",
    "    if self.rank_fitness:\n",
    "      reward_table = compute_centered_ranks(reward_table)\n",
    "    \n",
    "    if self.weight_decay > 0:\n",
    "      l2_decay = compute_weight_decay(self.weight_decay, self.solutions)\n",
    "      reward_table += l2_decay\n",
    "\n",
    "    reward_offset = 1\n",
    "    if self.average_baseline:\n",
    "      b = np.mean(reward_table)\n",
    "      reward_offset = 0\n",
    "    else:\n",
    "      b = reward_table[0] # baseline\n",
    "      \n",
    "    reward = reward_table[reward_offset:]\n",
    "    idx = np.argsort(reward)[::-1]\n",
    "\n",
    "    best_reward = reward[idx[0]]\n",
    "    if (best_reward > b or self.average_baseline):\n",
    "      best_mu = self.mu + self.epsilon_full[idx[0]]\n",
    "      best_reward = reward[idx[0]]\n",
    "    else:\n",
    "      best_mu = self.mu\n",
    "      best_reward = b\n",
    "\n",
    "    self.curr_best_reward = best_reward\n",
    "    self.curr_best_mu = best_mu\n",
    "\n",
    "    if self.first_interation:\n",
    "      self.first_interation = False\n",
    "      self.best_reward = self.curr_best_reward\n",
    "      self.best_mu = best_mu\n",
    "    else:\n",
    "      if self.forget_best or (self.curr_best_reward > self.best_reward):\n",
    "        self.best_mu = best_mu\n",
    "        self.best_reward = self.curr_best_reward\n",
    "\n",
    "    # adaptive sigma\n",
    "    # normalization\n",
    "    stdev_reward = reward.std()\n",
    "    epsilon = self.epsilon\n",
    "    sigma = self.sigma\n",
    "    S = ((epsilon * epsilon - (sigma * sigma).reshape(1, self.num_params)) / sigma.reshape(1, self.num_params))\n",
    "    reward_avg = (reward[:self.batch_size] + reward[self.batch_size:]) / 2.0\n",
    "    rS = reward_avg - b\n",
    "    delta_sigma = (np.dot(rS, S)) / (2 * self.batch_size * stdev_reward)\n",
    "\n",
    "    # move mean to the average of the best idx means\n",
    "    rT = (reward[:self.batch_size] - reward[self.batch_size:])\n",
    "    change_mu = self.learning_rate * np.dot(rT, epsilon)\n",
    "    self.mu += change_mu\n",
    "\n",
    "    # adjust sigma according to the adaptive sigma calculation\n",
    "    change_sigma = self.sigma_alpha * delta_sigma\n",
    "    change_sigma = np.minimum(change_sigma, self.sigma)\n",
    "    change_sigma = np.maximum(change_sigma, - 0.5 * self.sigma)\n",
    "    self.sigma += change_sigma\n",
    "    self.sigma[self.sigma > self.sigma_limit] *= self.sigma_decay\n",
    "    \n",
    "    if (self.learning_rate > self.learning_rate_limit):\n",
    "      self.learning_rate *= self.learning_rate_decay\n",
    "\n",
    "  def done(self):\n",
    "    return (self.rms_stdev() < self.done_threshold)\n",
    "\n",
    "  def current_param(self):\n",
    "    return self.curr_best_mu\n",
    "  \n",
    "  def best_param(self):\n",
    "    return self.best_mu\n",
    "\n",
    "  def result(self): # return best params so far, along with historically best reward, curr reward, sigma\n",
    "    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Args = namedtuple('Args', ['batch_size', 'test_batch_size', 'epochs', 'lr', 'cuda', 'seed', 'log_interval'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = Args(batch_size=1000, test_batch_size=1000, epochs=30, lr=0.001, cuda=True, seed=0, log_interval=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "  torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "valid_loader = train_loader\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "  datasets.MNIST('MNIST_data', train=False, transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))])),\n",
    "  batch_size=args.batch_size, shuffle=True, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "  def __init__(self):\n",
    "    super(Net, self).__init__()\n",
    "    self.num_filter1 = 8\n",
    "    self.num_filter2 = 16\n",
    "    self.num_padding = 2\n",
    "    # input is 28x28\n",
    "    # padding=2 for same padding\n",
    "    self.conv1 = nn.Conv2d(1, self.num_filter1, 5, padding=self.num_padding)\n",
    "    # feature map size is 14*14 by pooling\n",
    "    # padding=2 for same padding\n",
    "    self.conv2 = nn.Conv2d(self.num_filter1, self.num_filter2, 5, padding=self.num_padding)\n",
    "    # feature map size is 7*7 by pooling\n",
    "    self.fc = nn.Linear(self.num_filter2*7*7, 10)\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "    x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "    x = x.view(-1, self.num_filter2*7*7)   # reshape Variable\n",
    "    x = self.fc(x)\n",
    "    return F.log_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NPOPULATION = 101\n",
    "weight_decay_coef = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "models = []\n",
    "for i in range(NPOPULATION):\n",
    "  model = Net()\n",
    "  if args.cuda:\n",
    "    model.cuda()\n",
    "  model.eval()\n",
    "  models.append(model)\n",
    "'''\n",
    "\n",
    "model = Net()\n",
    "if args.cuda:\n",
    "  model.cuda()\n",
    "\n",
    "orig_model = copy.deepcopy(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11274\n"
     ]
    }
   ],
   "source": [
    "# get init params\n",
    "orig_params = []\n",
    "model_shapes = []\n",
    "for param in orig_model.parameters():\n",
    "  p = param.data.cpu().numpy()\n",
    "  model_shapes.append(p.shape)\n",
    "  orig_params.append(p.flatten())\n",
    "orig_params_flat = np.concatenate(orig_params)\n",
    "NPARAMS = len(orig_params_flat)\n",
    "print(NPARAMS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# NPARAMS = 11274"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_model(flat_param, model, model_shapes):\n",
    "  idx = 0\n",
    "  i = 0\n",
    "  for param in model.parameters():\n",
    "    delta = np.product(model_shapes[i])\n",
    "    block = flat_param[idx:idx+delta]\n",
    "    block = np.reshape(block, model_shapes[i])\n",
    "    i += 1\n",
    "    idx += delta\n",
    "    block_data = torch.from_numpy(block).float()\n",
    "    if args.cuda:\n",
    "      block_data = block_data.cuda()\n",
    "    param.data = block_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(model, test_loader, print_mode=True, return_loss=False):\n",
    "  model.eval()\n",
    "  test_loss = 0\n",
    "  correct = 0\n",
    "  for data, target in test_loader:\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data, volatile=True), Variable(target)\n",
    "    output = model(data)\n",
    "    test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss\n",
    "    pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
    "    correct += pred.eq(target.data.view_as(pred)).cpu().sum()\n",
    "\n",
    "  test_loss /= len(test_loader.dataset)\n",
    "  acc = correct / len(test_loader.dataset)\n",
    "  \n",
    "  if print_mode:\n",
    "    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\\n'.format(\n",
    "      test_loss, correct, len(test_loader.dataset),\n",
    "      100. * acc))\n",
    "  \n",
    "  if return_loss:\n",
    "    return test_loss\n",
    "  return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "es = SimpleES(NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.999,\n",
    "              sigma_alpha=0.2,\n",
    "              sigma_limit=0.001,\n",
    "              elite_ratio=0.1,\n",
    "              average_baseline=False,\n",
    "              forget_best=True\n",
    "             )\n",
    "es = OpenES(   NPARAMS,      # number of model parameters\n",
    "               popsize=NPOPULATION,\n",
    "               sigma_init=0.01,       # initial standard deviation\n",
    "               sigma_decay=0.999,     # anneal standard deviation\n",
    "               sigma_limit=0.01,\n",
    "               antithetic=True,\n",
    "             )\n",
    "es = SimpleGA(NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.999,\n",
    "              sigma_limit=0.001\n",
    "             )\n",
    "\"\"\"\n",
    "start_time = time.time()\n",
    "\n",
    "es = PEPG(    NPARAMS,\n",
    "              popsize=NPOPULATION,\n",
    "              sigma_init=0.01,\n",
    "              sigma_decay=0.999,\n",
    "              sigma_alpha=0.2,\n",
    "              sigma_limit=0.01,\n",
    "              learning_rate=0.1,            # learning rate for standard deviation\n",
    "              learning_rate_decay = 0.9999, # annealing the learning rate\n",
    "              learning_rate_limit = 0.01,   # stop annealing learning rate\n",
    "              average_baseline=False,\n",
    "             )\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init time 0.0007948875427246094\n"
     ]
    }
   ],
   "source": [
    "print('init time', end_time-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def worker(procnum, model, solution, data, target, send_end):\n",
    "  update_model(solution, model, model_shapes)\n",
    "  output = model(data)\n",
    "  loss = F.nll_loss(output, target)\n",
    "  reward = - loss.data[0]\n",
    "  send_end.send(reward)\n",
    "\n",
    "def batch_simulation(model_list, solutions, data, target, process_count):\n",
    "  jobs = []\n",
    "  pipe_list = []\n",
    "\n",
    "  for i in range(process_count):\n",
    "    recv_end, send_end = mp.Pipe(False)\n",
    "    p = mp.Process(target=worker, args=(i, model_list[i], solutions[i], data, target, send_end))\n",
    "    jobs.append(p)\n",
    "    pipe_list.append(recv_end)\n",
    "\n",
    "  for p in jobs:\n",
    "    p.start()\n",
    "\n",
    "  for p in jobs:\n",
    "    p.join()\n",
    "\n",
    "  result_list = [x.recv() for x in pipe_list]\n",
    "  return np.array(result_list)\n",
    "\n",
    "\n",
    "def batch_simulation_sequential(model_list, solutions, data, target, process_count):\n",
    "  result_list = []\n",
    "  for i in range(process_count):\n",
    "    update_model(solutions[i], model_list[i], model_shapes)\n",
    "    output = model_list[i](data)\n",
    "    loss = F.nll_loss(output, target)\n",
    "    reward = - loss.data[0]\n",
    "    result_list.append(reward)\n",
    "  return np.array(result_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 -2.3015370369 -1.3625762887e-05 0.00999493357973\n",
      "1 10 -2.29273867607 6.49126070597e-05 0.00994640534138\n",
      "1 20 -2.20771574974 0.000668892864525 0.00990713349438\n",
      "1 30 -1.93725275993 0.00101594478195 0.00986611657684\n",
      "1 40 -1.55533969402 0.00124605839688 0.00983399961559\n",
      "1 50 -1.20898079872 0.00166836133973 0.00979191068696\n",
      "valid_acc 74.89166666666667\n",
      "best valid_acc 74.89166666666667\n",
      "2 0 -1.02994275093 0.00186914703878 0.00975176003435\n",
      "2 10 -0.945468187332 0.00202775603505 0.00969804607107\n",
      "2 20 -0.752176880836 0.00210628802819 0.00964647891847\n",
      "2 30 -0.725091934204 0.00219084125642 0.00960169752304\n",
      "2 40 -0.700149953365 0.00213861995048 0.0095581890539\n",
      "2 50 -0.579843878746 0.00228639487486 0.00951103984905\n",
      "valid_acc 83.84166666666667\n",
      "best valid_acc 83.84166666666667\n",
      "3 0 -0.534947991371 0.00239417188268 0.00948159932136\n",
      "3 10 -0.588443875313 0.00249070259302 0.00941749535365\n",
      "3 20 -0.556837201118 0.00261357393045 0.00936498248429\n",
      "3 30 -0.452707111835 0.00258899662774 0.00932860551505\n",
      "3 40 -0.48762050271 0.00259752851086 0.00927083251153\n",
      "3 50 -0.466498196125 0.00259642977417 0.00920004533465\n",
      "valid_acc 86.265\n",
      "best valid_acc 86.265\n",
      "4 0 -0.477015793324 0.00256985328985 0.00915108006735\n",
      "4 10 -0.437452077866 0.00249764138551 0.00910470903234\n",
      "4 20 -0.385276913643 0.00254663529785 0.00905435457787\n",
      "4 30 -0.423624098301 0.00246848767799 0.00898625109022\n",
      "4 40 -0.420540690422 0.00248314869051 0.00892808886835\n",
      "4 50 -0.402457535267 0.0022960201622 0.00887009107729\n",
      "valid_acc 88.03333333333333\n",
      "best valid_acc 88.03333333333333\n",
      "5 0 -0.388254404068 0.00229812033202 0.00883767569532\n",
      "5 10 -0.36739128828 0.00225515689787 0.00879538684326\n",
      "5 20 -0.428320139647 0.00232945911414 0.00875153921834\n",
      "5 30 -0.383684903383 0.00218269981679 0.00870113507805\n",
      "5 40 -0.432154774666 0.00211687321264 0.0086407168929\n",
      "5 50 -0.390559136868 0.00225944899588 0.00857692081228\n",
      "valid_acc 89.45666666666666\n",
      "best valid_acc 89.45666666666666\n",
      "6 0 -0.350162833929 0.00217606557115 0.00853646823665\n",
      "6 10 -0.376082777977 0.00216022693157 0.00849116771494\n",
      "6 20 -0.323433965445 0.00219216193793 0.00843914685768\n",
      "6 30 -0.339197009802 0.00218088411865 0.00838950164032\n",
      "6 40 -0.357838600874 0.00213946051877 0.00832941052172\n",
      "6 50 -0.307298094034 0.00231204224598 0.00828102141417\n",
      "valid_acc 90.27\n",
      "best valid_acc 90.27\n",
      "7 0 -0.317924231291 0.00223062203635 0.00822577723894\n",
      "7 10 -0.285778015852 0.00236541841039 0.00819597635313\n",
      "7 20 -0.284006983042 0.00198477741541 0.00814251824586\n",
      "7 30 -0.322694391012 0.00188935074482 0.00808718646583\n",
      "7 40 -0.27657687664 0.00197277407865 0.00804459896425\n",
      "7 50 -0.301543593407 0.00205474232989 0.00800435444149\n",
      "valid_acc 91.33666666666667\n",
      "best valid_acc 91.33666666666667\n",
      "8 0 -0.283228337765 0.00203809646956 0.00797065817268\n",
      "8 10 -0.283903241158 0.00194159524936 0.00793005655767\n",
      "8 20 -0.283037543297 0.00191677193605 0.00788335025407\n",
      "8 30 -0.290317207575 0.00193486343986 0.00784347825599\n",
      "8 40 -0.265616208315 0.00190065085014 0.00781388655411\n",
      "8 50 -0.287095457315 0.00186444017786 0.00776820602259\n",
      "valid_acc 91.97166666666666\n",
      "best valid_acc 91.97166666666666\n",
      "9 0 -0.279303252697 0.00181614868304 0.00773340583823\n",
      "9 10 -0.295207768679 0.00196544461349 0.00769751001031\n",
      "9 20 -0.229951828718 0.00181948822975 0.00766496465422\n",
      "9 30 -0.234405517578 0.00182119278696 0.00760823026436\n",
      "9 40 -0.250581771135 0.00166729777559 0.00757545657994\n",
      "9 50 -0.27878895402 0.00169749509708 0.0075298828501\n",
      "valid_acc 92.53833333333333\n",
      "best valid_acc 92.53833333333333\n",
      "10 0 -0.255490064621 0.00162704056657 0.0074867963852\n",
      "10 10 -0.217073246837 0.00160804824496 0.0074384021698\n",
      "10 20 -0.249490201473 0.00161058284927 0.00740082490856\n",
      "10 30 -0.231452286243 0.00162493850201 0.00735639284748\n",
      "10 40 -0.283614039421 0.0018873890804 0.00731254288374\n",
      "10 50 -0.202628836036 0.00179595386168 0.00725622318866\n",
      "valid_acc 92.98666666666666\n",
      "best valid_acc 92.98666666666666\n",
      "11 0 -0.228246167302 0.00185595304637 0.00721301971792\n",
      "11 10 -0.246928885579 0.00190782238076 0.00718078882934\n",
      "11 20 -0.250431805849 0.00173171655879 0.0071363476788\n",
      "11 30 -0.236849501729 0.00181300292912 0.00708508880735\n",
      "11 40 -0.211556985974 0.00187782002318 0.00706362938696\n",
      "11 50 -0.250708609819 0.00175434221584 0.00702612942197\n",
      "valid_acc 93.23166666666667\n",
      "best valid_acc 93.23166666666667\n",
      "12 0 -0.247138738632 0.00160160840657 0.00698477796163\n",
      "12 10 -0.201228529215 0.00172650950514 0.00692577066559\n",
      "12 20 -0.257842421532 0.00180919927104 0.00688569783683\n",
      "12 30 -0.269445091486 0.00173720811415 0.00684203242737\n",
      "12 40 -0.223444715142 0.00155992023688 0.00680660576905\n",
      "12 50 -0.22124402225 0.00166730077266 0.00677689358591\n",
      "valid_acc 93.44833333333334\n",
      "best valid_acc 93.44833333333334\n",
      "13 0 -0.214715898037 0.00179637737642 0.00673256274863\n",
      "13 10 -0.234049871564 0.00169248353386 0.00668281554243\n",
      "13 20 -0.222717136145 0.0018210551148 0.00664488739897\n",
      "13 30 -0.196537688375 0.00160163345886 0.00661051698536\n",
      "13 40 -0.192704156041 0.00168987255665 0.0065717295929\n",
      "13 50 -0.239279657602 0.00181448715169 0.00655409701373\n",
      "valid_acc 93.78\n",
      "best valid_acc 93.78\n",
      "14 0 -0.241502359509 0.00168230898669 0.00653021922311\n",
      "14 10 -0.202527299523 0.00154452262961 0.0064784422372\n",
      "14 20 -0.17320817709 0.00153281650973 0.00644997898052\n",
      "14 30 -0.226853609085 0.00143728849548 0.0064220583614\n",
      "14 40 -0.18000896275 0.00162157542925 0.00638457212854\n",
      "14 50 -0.189259782434 0.00173029026986 0.00634249501971\n",
      "valid_acc 94.10833333333333\n",
      "best valid_acc 94.10833333333333\n",
      "15 0 -0.203682333231 0.00171083056476 0.00631216941869\n",
      "15 10 -0.187640994787 0.00164813804619 0.00629653885912\n",
      "15 20 -0.164975613356 0.00177802857191 0.00626794273906\n",
      "15 30 -0.184161305428 0.00159335852653 0.00624137012062\n",
      "15 40 -0.198896929622 0.0014962170437 0.00623708513843\n",
      "15 50 -0.148714929819 0.00153750646892 0.00620636466898\n",
      "valid_acc 94.13\n",
      "best valid_acc 94.13\n",
      "16 0 -0.205399557948 0.0012951463989 0.00617187675095\n",
      "16 10 -0.173367708921 0.00128017314244 0.00613150017287\n",
      "16 20 -0.198582842946 0.00130505093479 0.00609943391596\n",
      "16 30 -0.165766522288 0.0013833225837 0.00607156006282\n",
      "16 40 -0.130758330226 0.00121558669369 0.00605021441079\n",
      "16 50 -0.177086427808 0.000984930712074 0.00600680909679\n",
      "valid_acc 94.63333333333334\n",
      "best valid_acc 94.63333333333334\n",
      "17 0 -0.177764698863 0.00134585874721 0.0059875821467\n",
      "17 10 -0.143626958132 0.00119224786603 0.00596928168881\n",
      "17 20 -0.141031444073 0.0012797151177 0.00594968314532\n",
      "17 30 -0.174031272531 0.00124694237002 0.00591305367993\n",
      "17 40 -0.167387768626 0.00118592691112 0.00587774804562\n",
      "17 50 -0.218059495091 0.00121502436347 0.00586143348668\n",
      "valid_acc 94.71666666666667\n",
      "best valid_acc 94.71666666666667\n",
      "18 0 -0.148777604103 0.00131193322655 0.00583270227887\n",
      "18 10 -0.166642814875 0.00114690493126 0.00581041180846\n",
      "18 20 -0.186952114105 0.00121549549845 0.0057782466257\n",
      "18 30 -0.15556588769 0.00122752032368 0.00574015300794\n",
      "18 40 -0.172016680241 0.0012493801262 0.00571581099189\n",
      "18 50 -0.137878060341 0.00131641035889 0.00568227689086\n",
      "valid_acc 94.95333333333333\n",
      "best valid_acc 94.95333333333333\n",
      "19 0 -0.146104156971 0.00120955651369 0.00565169476766\n",
      "19 10 -0.198827937245 0.00141291048931 0.00562262071709\n",
      "19 20 -0.144020780921 0.00154895079385 0.00560796990616\n",
      "19 30 -0.154505193233 0.00169510530994 0.00556795175492\n",
      "19 40 -0.14247302711 0.001748578763 0.00554667657647\n",
      "19 50 -0.141776800156 0.00180491148413 0.00552286419132\n",
      "valid_acc 95.205\n",
      "best valid_acc 95.205\n",
      "20 0 -0.146836578846 0.00167703830221 0.00549648485563\n",
      "20 10 -0.138202950358 0.00154670039383 0.00548004573553\n",
      "20 20 -0.184400141239 0.00152600296823 0.00545366490062\n",
      "20 30 -0.143896520138 0.00160661668659 0.00543165118697\n",
      "20 40 -0.175112649798 0.00172598462062 0.00539599507578\n",
      "20 50 -0.119840249419 0.00154576443407 0.0053683516701\n",
      "valid_acc 95.62333333333333\n",
      "best valid_acc 95.62333333333333\n",
      "21 0 -0.14806997776 0.00168400541571 0.0053501100737\n",
      "21 10 -0.140091404319 0.00160504614447 0.00531015889016\n",
      "21 20 -0.130147084594 0.00160213152406 0.00528674630605\n",
      "21 30 -0.154827266932 0.00149827744721 0.00525816371571\n",
      "21 40 -0.124978497624 0.00152841931547 0.00523408121851\n",
      "21 50 -0.156943887472 0.00155619766655 0.00520790130635\n",
      "valid_acc 95.32666666666667\n",
      "22 0 -0.173790097237 0.00172289916085 0.00519213474151\n",
      "22 10 -0.128586992621 0.0016886308845 0.00516810229852\n",
      "22 20 -0.150675520301 0.00173865741996 0.00514251304142\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 30 -0.18594917655 0.00171536419735 0.00510050577734\n",
      "22 40 -0.130576968193 0.00167346014895 0.00508697490512\n",
      "22 50 -0.176683276892 0.00169889200255 0.00506609005525\n",
      "valid_acc 95.67166666666667\n",
      "best valid_acc 95.67166666666667\n",
      "23 0 -0.123710893095 0.00175887741862 0.00504928099572\n",
      "23 10 -0.148195341229 0.00156132246357 0.00501304154251\n",
      "23 20 -0.152287974954 0.00153527711225 0.00497958675176\n",
      "23 30 -0.16320194304 0.00129272222605 0.00496548395933\n",
      "23 40 -0.133498221636 0.00147776282228 0.00494285211434\n",
      "23 50 -0.138774454594 0.00147879835235 0.00492893151602\n",
      "valid_acc 95.66666666666667\n",
      "24 0 -0.143809661269 0.00156957833378 0.00490763531684\n",
      "24 10 -0.124419309199 0.00144986556885 0.00488567931825\n",
      "24 20 -0.140805080533 0.00149847732212 0.00485429002956\n",
      "24 30 -0.106018573046 0.00141167055513 0.00483692717515\n",
      "24 40 -0.116842091084 0.00131821749433 0.00481588374114\n",
      "24 50 -0.110816329718 0.00133028239606 0.00478639111557\n",
      "valid_acc 96.02333333333334\n",
      "best valid_acc 96.02333333333334\n",
      "25 0 -0.159575849771 0.00137692057491 0.0047658879135\n",
      "25 10 -0.133688047528 0.0013833476154 0.00474327288848\n",
      "25 20 -0.108072891831 0.00126796734447 0.00471332464723\n",
      "25 30 -0.120328903198 0.00122464149458 0.00469300210782\n",
      "25 40 -0.125829622149 0.00126145593631 0.00467960737569\n",
      "25 50 -0.141977772117 0.00132713181665 0.00466025034871\n",
      "valid_acc 96.08166666666666\n",
      "best valid_acc 96.08166666666666\n",
      "26 0 -0.128144487739 0.00123311464139 0.0046514291662\n",
      "26 10 -0.125442802906 0.00111283540272 0.00463212334944\n",
      "26 20 -0.13814291358 0.000976153462259 0.00462397255841\n",
      "26 30 -0.134485304356 0.00119751321698 0.0045903791726\n",
      "26 40 -0.11636903137 0.0011160751882 0.00457235319571\n",
      "26 50 -0.122194960713 0.00114406087639 0.00454908884765\n",
      "valid_acc 96.00999999999999\n",
      "27 0 -0.124889291823 0.0011201247686 0.00452502091071\n",
      "27 10 -0.123839326203 0.00101655327753 0.00450688188045\n",
      "27 20 -0.111128486693 0.00109660191301 0.00448823224412\n",
      "27 30 -0.114916615188 0.00121730598323 0.00447517049183\n",
      "27 40 -0.128138899803 0.00110000243883 0.00445165795566\n",
      "27 50 -0.142946183681 0.00113777595335 0.00444716161661\n",
      "valid_acc 96.3\n",
      "best valid_acc 96.3\n",
      "28 0 -0.0771485492587 0.00106669481591 0.004420195888\n",
      "28 10 -0.115099839866 0.00110934501396 0.00439348348317\n",
      "28 20 -0.0974150672555 0.000970675311077 0.00436644196771\n",
      "28 30 -0.104851402342 0.00115356691446 0.0043557045474\n",
      "28 40 -0.0992491543293 0.00102187330095 0.00433801918758\n",
      "28 50 -0.154656141996 0.000922835777842 0.00431815452155\n",
      "valid_acc 96.465\n",
      "best valid_acc 96.465\n",
      "29 0 -0.110808476806 0.000935465203041 0.00429304041465\n",
      "29 10 -0.0713839754462 0.0010944629372 0.00428607181398\n",
      "29 20 -0.146833896637 0.00112079785318 0.00427141123857\n",
      "29 30 -0.107853420079 0.00123810584388 0.00425739700635\n",
      "29 40 -0.121415421367 0.00108632022071 0.00423585662346\n",
      "29 50 -0.0819023102522 0.00120421669081 0.00421383565347\n",
      "valid_acc 96.50666666666666\n",
      "best valid_acc 96.50666666666666\n",
      "30 0 -0.0928794592619 0.00114407587364 0.00419099853528\n",
      "30 10 -0.149121239781 0.00110120434331 0.0041713923168\n",
      "30 20 -0.0905337557197 0.00105364304058 0.00414727786489\n",
      "30 30 -0.100121870637 0.00113780513286 0.00414119506585\n",
      "30 40 -0.1223333776 0.00108560839877 0.00413111987588\n",
      "30 50 -0.139318734407 0.0010752668621 0.00412115387288\n",
      "valid_acc 96.38166666666666\n"
     ]
    }
   ],
   "source": [
    "#'''\n",
    "best_valid_acc = 0\n",
    "training_log = []\n",
    "for epoch in range(1, 1*args.epochs + 1):\n",
    "\n",
    "  # train loop\n",
    "  model.eval()\n",
    "  for batch_idx, (data, target) in enumerate(train_loader):\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data), Variable(target)\n",
    "    \n",
    "    solutions = es.ask()\n",
    "    reward = np.zeros(es.popsize)\n",
    "    \n",
    "    for i in range(es.popsize):\n",
    "      update_model(solutions[i], model, model_shapes)\n",
    "      output = model(data)\n",
    "      loss = F.nll_loss(output, target)\n",
    "      reward[i] = - loss.data[0]\n",
    "\n",
    "    best_raw_reward = reward.max()\n",
    "\n",
    "    es.tell(reward)\n",
    "\n",
    "    result = es.result()\n",
    "    \n",
    "    if (batch_idx % 10 == 0):\n",
    "      print(epoch, batch_idx, best_raw_reward, result[0].mean(), es.rms_stdev())\n",
    "\n",
    "  curr_solution = es.current_param()\n",
    "  update_model(curr_solution, model, model_shapes)\n",
    "\n",
    "  valid_acc = evaluate(model, valid_loader, print_mode=False)\n",
    "  training_log.append([epoch, valid_acc])\n",
    "  print('valid_acc', valid_acc * 100.)\n",
    "  if valid_acc >= best_valid_acc:\n",
    "    best_valid_acc = valid_acc\n",
    "    best_model = copy.deepcopy(model)\n",
    "    print('best valid_acc', best_valid_acc * 100.)\n",
    "#'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 0 -0.133420318365 0.00100647542086 0.00410479424588\n",
      "1 10 -0.0963906720281 0.00101848202417 0.0040793648565\n",
      "1 20 -0.12268974632 0.00110729788891 0.00406207562388\n",
      "1 30 -0.0774867087603 0.00103202388317 0.00404572607528\n",
      "1 40 -0.110574513674 0.00109495313029 0.00403260798392\n",
      "1 50 -0.120058402419 0.000908460687557 0.00401968590472\n",
      "valid_acc 96.47666666666666\n",
      "2 0 -0.0982737392187 0.000913908051058 0.00400306431467\n",
      "2 10 -0.0826506316662 0.00095355464919 0.00399272188401\n",
      "2 20 -0.0968995243311 0.000939595503046 0.00397913546538\n",
      "2 30 -0.0916180536151 0.000905829544027 0.00396826501974\n",
      "2 40 -0.100446596742 0.000948154835541 0.00395949369288\n",
      "2 50 -0.11905310303 0.000888758552033 0.00394812686508\n",
      "valid_acc 96.66666666666667\n",
      "best valid_acc 96.66666666666667\n",
      "3 0 -0.121412277222 0.000843183125303 0.00393802627524\n",
      "3 10 -0.0769304484129 0.000904433437317 0.00391759855502\n",
      "3 20 -0.0717141255736 0.000935867888462 0.00390324359036\n",
      "3 30 -0.118696071208 0.000800283276926 0.00389216911056\n",
      "3 40 -0.0934861376882 0.000889597969688 0.00388495732114\n",
      "3 50 -0.0841078013182 0.000818508930957 0.00387083970762\n",
      "valid_acc 96.69\n",
      "best valid_acc 96.69\n",
      "4 0 -0.104738637805 0.000662866614839 0.00385961326847\n",
      "4 10 -0.0727778226137 0.00074135527529 0.00384495678846\n",
      "4 20 -0.101395905018 0.000776288844157 0.00383040370787\n",
      "4 30 -0.102986305952 0.000572479397929 0.00381530973364\n",
      "4 40 -0.122906856239 0.000731725395055 0.00380346058975\n",
      "4 50 -0.0769169777632 0.000783253671179 0.00379927951226\n",
      "valid_acc 96.67\n",
      "5 0 -0.0780416727066 0.000713250362582 0.00378323443046\n",
      "5 10 -0.0806866586208 0.000861353883914 0.00376864531953\n",
      "5 20 -0.0795269757509 0.000739684070131 0.0037594109615\n",
      "5 30 -0.134278312325 0.00083583152241 0.00375275161931\n",
      "5 40 -0.138093292713 0.000983117423222 0.00373513174461\n",
      "5 50 -0.107232280076 0.000910562194792 0.00371619993869\n",
      "valid_acc 96.75333333333333\n",
      "best valid_acc 96.75333333333333\n",
      "6 0 -0.105698496103 0.00096022615292 0.0037040463898\n",
      "6 10 -0.106836400926 0.00100941367848 0.00368848218059\n",
      "6 20 -0.0910117700696 0.00103888822197 0.00367493861566\n",
      "6 30 -0.112654075027 0.00104590881413 0.00366612125905\n",
      "6 40 -0.0967606157064 0.00102788524022 0.00364758022364\n",
      "6 50 -0.0981357097626 0.00101602336317 0.0036347299994\n",
      "valid_acc 96.87833333333333\n",
      "best valid_acc 96.87833333333333\n",
      "7 0 -0.106010444462 0.00111367571843 0.00362775965205\n",
      "7 10 -0.102175667882 0.00113681701888 0.00361116210933\n",
      "7 20 -0.0681428536773 0.00112282416519 0.00359762351432\n",
      "7 30 -0.0999428555369 0.00113082783383 0.00358199935772\n",
      "7 40 -0.0899093002081 0.00108036206728 0.00357126908187\n",
      "7 50 -0.10363214463 0.00113993451612 0.0035569434788\n",
      "valid_acc 96.89833333333333\n",
      "best valid_acc 96.89833333333333\n",
      "8 0 -0.0868514254689 0.00113612600623 0.00354999161785\n",
      "8 10 -0.111564956605 0.00117583188516 0.00353546500392\n",
      "8 20 -0.113716006279 0.00119112174132 0.00352937917261\n",
      "8 30 -0.0885119959712 0.00125341256 0.0035151487036\n",
      "8 40 -0.109584592283 0.0012628922389 0.00350538662091\n",
      "8 50 -0.0903675332665 0.00117750698483 0.00349280545627\n",
      "valid_acc 96.91666666666666\n",
      "best valid_acc 96.91666666666666\n",
      "9 0 -0.120133377612 0.00124462281184 0.00347943430786\n",
      "9 10 -0.174418032169 0.00127189523421 0.00346566902411\n",
      "9 20 -0.1269223243 0.00117058758585 0.00346423987129\n",
      "9 30 -0.0888032391667 0.00119877872278 0.00344825600223\n",
      "9 40 -0.0758956447244 0.00120006770934 0.00343812606766\n",
      "9 50 -0.098711438477 0.00115555717821 0.00341870805784\n",
      "valid_acc 96.96166666666667\n",
      "best valid_acc 96.96166666666667\n",
      "10 0 -0.0909817144275 0.00115131177764 0.00340807798618\n",
      "10 10 -0.114620372653 0.00112263442455 0.00339249543137\n",
      "10 20 -0.0920019671321 0.00109921764645 0.00337931469249\n",
      "10 30 -0.10684799403 0.0009468329089 0.00337695517202\n",
      "10 40 -0.121073998511 0.000898825070837 0.00336625013195\n",
      "10 50 -0.114142149687 0.0009582173953 0.0033630978411\n",
      "valid_acc 96.91166666666666\n",
      "11 0 -0.0873347967863 0.000902955442807 0.00335764969979\n",
      "11 10 -0.124757565558 0.000767529468536 0.0033457814097\n",
      "11 20 -0.105243451893 0.00071801944591 0.00333394019578\n",
      "11 30 -0.0848548561335 0.000714599585306 0.00332258074051\n",
      "11 40 -0.0751351267099 0.000647725632998 0.00330933239278\n",
      "11 50 -0.062558196485 0.000676834855723 0.00329714617438\n",
      "valid_acc 96.92666666666668\n",
      "12 0 -0.091876655817 0.000678327779487 0.00328361745151\n",
      "12 10 -0.0622350163758 0.00069562844212 0.0032776560951\n",
      "12 20 -0.107692480087 0.000780777486769 0.00326302334042\n",
      "12 30 -0.100586794317 0.000808636414056 0.00324683109376\n",
      "12 40 -0.0938239470124 0.000751731979001 0.00323460317442\n",
      "12 50 -0.0935280546546 0.000824563670148 0.00322407892103\n",
      "valid_acc 97.04833333333333\n",
      "best valid_acc 97.04833333333333\n",
      "13 0 -0.126834049821 0.000928358963003 0.00321070199538\n",
      "13 10 -0.0741639956832 0.000814800982735 0.00320132849411\n",
      "13 20 -0.109093934298 0.000841955900339 0.00319496063707\n",
      "13 30 -0.121440917253 0.000799832549388 0.00317674311603\n",
      "13 40 -0.07275108248 0.00083736628366 0.00317137162089\n",
      "13 50 -0.0720087066293 0.000590339543898 0.00316425540063\n",
      "valid_acc 96.90833333333333\n",
      "14 0 -0.068630002439 0.000647449010022 0.00315712611185\n",
      "14 10 -0.113144636154 0.000712758675454 0.00314915603671\n",
      "14 20 -0.0965237021446 0.000710908029592 0.00313837291398\n",
      "14 30 -0.0937501341105 0.000699625587473 0.00313332126441\n",
      "14 40 -0.087478287518 0.000889381731809 0.00312463809603\n",
      "14 50 -0.109475709498 0.000822526144248 0.00311558782996\n",
      "valid_acc 97.11\n",
      "best valid_acc 97.11\n",
      "15 0 -0.0921492651105 0.00080792763754 0.00310341768609\n",
      "15 10 -0.0793410167098 0.000937948016001 0.00309252396014\n",
      "15 20 -0.0774940922856 0.000874527546045 0.0030851333997\n",
      "15 30 -0.0809038430452 0.000816788785752 0.00307863165174\n",
      "15 40 -0.0932660475373 0.000835907809712 0.00306984374528\n",
      "15 50 -0.105781376362 0.000913642340341 0.00306768884014\n",
      "valid_acc 97.14500000000001\n",
      "best valid_acc 97.14500000000001\n",
      "16 0 -0.0815956145525 0.000830251654488 0.00306520546294\n",
      "16 10 -0.0860729664564 0.000766246401879 0.00305972031462\n",
      "16 20 -0.0773474127054 0.000847525750308 0.00304991504242\n",
      "16 30 -0.0965083017945 0.000808166701085 0.00304063533981\n",
      "16 40 -0.123491629958 0.000815603176158 0.00303061711544\n",
      "16 50 -0.0757583975792 0.0007579876996 0.00302511560172\n",
      "valid_acc 97.12833333333334\n",
      "17 0 -0.0825085490942 0.000810230239308 0.00302216227537\n",
      "17 10 -0.100936844945 0.000714862813189 0.00301844427726\n",
      "17 20 -0.100155629218 0.000706238839837 0.00301073835167\n",
      "17 30 -0.0941829532385 0.000700833845068 0.00300262867555\n",
      "17 40 -0.10270319134 0.000732038859467 0.00298510432648\n",
      "17 50 -0.082995980978 0.000678203771916 0.00297843518316\n",
      "valid_acc 97.21499999999999\n",
      "best valid_acc 97.21499999999999\n",
      "18 0 -0.0698393061757 0.000725572449263 0.00296963733041\n",
      "18 10 -0.0699499920011 0.000704403223737 0.00296164592497\n",
      "18 20 -0.101439103484 0.000748321692599 0.00295336320224\n",
      "18 30 -0.0748478621244 0.000696365729988 0.00294788902309\n",
      "18 40 -0.105759516358 0.000796389126971 0.00294387229224\n",
      "18 50 -0.0686716809869 0.000799980343448 0.00293837737521\n",
      "valid_acc 97.195\n",
      "19 0 -0.0856649652123 0.000762679742039 0.00293103074105\n",
      "19 10 -0.101165525615 0.000766863462173 0.00292541837404\n",
      "19 20 -0.0697371661663 0.000855322252848 0.00292296218655\n",
      "19 30 -0.0662379711866 0.000779218714688 0.00291749457298\n",
      "19 40 -0.0742920935154 0.00083586279433 0.00291174261033\n",
      "19 50 -0.113724455237 0.00079082383892 0.00290501435986\n",
      "valid_acc 97.245\n",
      "best valid_acc 97.245\n",
      "20 0 -0.110401973128 0.000744738232473 0.00289348468036\n",
      "20 10 -0.111841656268 0.000744533842544 0.00288821112621\n",
      "20 20 -0.0661686733365 0.000751873776162 0.00288188946678\n",
      "20 30 -0.066940702498 0.000666129520325 0.00287758233138\n",
      "20 40 -0.0745821893215 0.00072557250156 0.00287355522569\n",
      "20 50 -0.0916078686714 0.000691483054633 0.00286862182801\n",
      "valid_acc 97.19666666666666\n",
      "21 0 -0.0759673789144 0.000591725454877 0.0028676485156\n",
      "21 10 -0.088543407619 0.000632907981836 0.00285969448278\n",
      "21 20 -0.0698254331946 0.000643858235298 0.00285509236396\n",
      "21 30 -0.0965352877975 0.000666586113059 0.00284856364684\n",
      "21 40 -0.0755655020475 0.000586839663368 0.00284008832513\n",
      "21 50 -0.0636272057891 0.00056963285479 0.00283591193074\n",
      "valid_acc 97.25333333333333\n",
      "best valid_acc 97.25333333333333\n",
      "22 0 -0.0863511487842 0.000615699047296 0.0028301248636\n",
      "22 10 -0.0910157412291 0.000566634121854 0.00282548563224\n",
      "22 20 -0.0751781463623 0.000688409779627 0.00282301892269\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 30 -0.0981971025467 0.000760815200301 0.00281621185794\n",
      "22 40 -0.0811848863959 0.000727178905861 0.00281083828747\n",
      "22 50 -0.114038132131 0.000734380403618 0.00280220744286\n",
      "valid_acc 97.31333333333333\n",
      "best valid_acc 97.31333333333333\n",
      "23 0 -0.0964765027165 0.000655040699541 0.00279404309991\n",
      "23 10 -0.0868964418769 0.000815731894 0.00278759889754\n",
      "23 20 -0.0750052630901 0.000772387613005 0.00278148350474\n",
      "23 30 -0.0984847769141 0.000740256872494 0.00277466742\n",
      "23 40 -0.0840069204569 0.000792859750229 0.00276823407172\n",
      "23 50 -0.0768686532974 0.000796182110385 0.00276240720782\n",
      "valid_acc 97.40666666666667\n",
      "best valid_acc 97.40666666666667\n",
      "24 0 -0.0898569822311 0.000665555343138 0.00275643284746\n",
      "24 10 -0.0746565312147 0.0007787089569 0.00275221930355\n",
      "24 20 -0.0942489132285 0.000822913810508 0.00274483697712\n",
      "24 30 -0.100115001202 0.000875561783549 0.0027386865403\n",
      "24 40 -0.0833004489541 0.000895611761626 0.00273319817884\n",
      "24 50 -0.049983587116 0.000828409837475 0.00272557113309\n",
      "valid_acc 97.44166666666668\n",
      "best valid_acc 97.44166666666668\n",
      "25 0 -0.0781069025397 0.00092110885121 0.00272639317862\n",
      "25 10 -0.0600539520383 0.0009750103088 0.00271816172613\n",
      "25 20 -0.0799731314182 0.000940920346499 0.00271447605412\n",
      "25 30 -0.0802579522133 0.000853283242829 0.00271378029859\n",
      "25 40 -0.0816368237138 0.000955469540543 0.00270607164509\n",
      "25 50 -0.084039658308 0.000878242384506 0.00270107513187\n",
      "valid_acc 97.39666666666666\n",
      "26 0 -0.0970829278231 0.000915011587067 0.00269469574687\n",
      "26 10 -0.0633664056659 0.000888214253737 0.00268193447577\n",
      "26 20 -0.097413636744 0.00092602301419 0.00267761302837\n",
      "26 30 -0.0619346983731 0.000823423222938 0.00267424850779\n",
      "26 40 -0.0802196860313 0.000814896330915 0.00266846931497\n",
      "26 50 -0.067788399756 0.000780551286447 0.00266362208298\n",
      "valid_acc 97.42166666666667\n",
      "27 0 -0.0642315074801 0.000759072942291 0.00265393840942\n",
      "27 10 -0.071975722909 0.00076511226207 0.00265118546154\n",
      "27 20 -0.0910146161914 0.000831893954537 0.00264330118895\n",
      "27 30 -0.0860832631588 0.00086254842242 0.00263403629705\n",
      "27 40 -0.0865575000644 0.000881195637472 0.00263214534762\n",
      "27 50 -0.0846612080932 0.00092380469447 0.00262839584025\n",
      "valid_acc 97.37666666666667\n",
      "28 0 -0.0585933998227 0.000819895745287 0.00262521690242\n",
      "28 10 -0.064607642591 0.000892668490011 0.00262285693074\n",
      "28 20 -0.0913728773594 0.000920521916696 0.00261663601835\n",
      "28 30 -0.08716596663 0.000845465898745 0.00261146071977\n",
      "28 40 -0.0709830448031 0.000858944773693 0.00260814786384\n",
      "28 50 -0.0834282115102 0.000828356007816 0.00260282191432\n",
      "valid_acc 97.435\n",
      "29 0 -0.0939118638635 0.000693606211115 0.00259914004608\n",
      "29 10 -0.100371494889 0.000792630399459 0.00259241093024\n",
      "29 20 -0.0572604425251 0.000680107837797 0.00258667990656\n",
      "29 30 -0.0875898525119 0.000783182386826 0.00258613706477\n",
      "29 40 -0.0706899240613 0.00077857477003 0.00258304675775\n",
      "29 50 -0.123943537474 0.000738485877819 0.00257810361391\n",
      "valid_acc 97.44666666666667\n",
      "best valid_acc 97.44666666666667\n",
      "30 0 -0.0625935047865 0.000720208367204 0.00256881129134\n",
      "30 10 -0.0537013038993 0.000696676362529 0.00256505226506\n",
      "30 20 -0.0539420470595 0.000739144239937 0.00256325650971\n",
      "30 30 -0.0672374516726 0.000710882674688 0.00256112642434\n",
      "30 40 -0.0539236776531 0.000693949993365 0.00255481542539\n",
      "30 50 -0.106714650989 0.000850658777249 0.00254824391973\n",
      "valid_acc 97.42166666666667\n",
      "31 0 -0.0618389286101 0.000741116906622 0.00254368536012\n",
      "31 10 -0.065222889185 0.000698541745111 0.00253907290806\n",
      "31 20 -0.0620870441198 0.000787148412144 0.0025314525653\n",
      "31 30 -0.0685837939382 0.000873634208639 0.00252903947848\n",
      "31 40 -0.0497282445431 0.000821183836304 0.00252200115766\n",
      "31 50 -0.0787934362888 0.000826065564397 0.00251784312884\n",
      "valid_acc 97.50666666666666\n",
      "best valid_acc 97.50666666666666\n",
      "32 0 -0.0684808194637 0.000760147708218 0.00251340921587\n",
      "32 10 -0.0882711410522 0.000714405530171 0.00251068341357\n",
      "32 20 -0.0733887702227 0.0007711657318 0.00251032612503\n",
      "32 30 -0.0677174702287 0.000780572970414 0.0025059947093\n",
      "32 40 -0.0732608437538 0.000745994545735 0.00250150063903\n",
      "32 50 -0.0599242709577 0.000822999098545 0.00250232411358\n",
      "valid_acc 97.43833333333333\n",
      "33 0 -0.0795421227813 0.000857622799219 0.00249536703723\n",
      "33 10 -0.053082883358 0.000855472408987 0.00249380030988\n",
      "33 20 -0.0733324736357 0.000813818529958 0.00248998328282\n",
      "33 30 -0.082282744348 0.000788717850934 0.00248537058032\n",
      "33 40 -0.0712718069553 0.000762714535954 0.00247771303934\n",
      "33 50 -0.0864481329918 0.000727643965297 0.00247765315545\n",
      "valid_acc 97.47166666666666\n",
      "34 0 -0.0802294239402 0.000806089622526 0.00247311893502\n",
      "34 10 -0.0806183665991 0.000786926922183 0.00246532127996\n",
      "34 20 -0.054039567709 0.000865365112911 0.00245876842543\n",
      "34 30 -0.093410231173 0.000902904830188 0.00245306034416\n",
      "34 40 -0.0570801943541 0.00086284793898 0.00244782977314\n",
      "34 50 -0.0608786195517 0.000860999657666 0.00244361640628\n",
      "valid_acc 97.49333333333333\n",
      "35 0 -0.0715939700603 0.000774853587566 0.00244144971938\n",
      "35 10 -0.0528771914542 0.000765654353147 0.0024335897789\n",
      "35 20 -0.0853509157896 0.000760809847663 0.00243066038424\n",
      "35 30 -0.0649507790804 0.000750688785208 0.00242552336423\n",
      "35 40 -0.0666754469275 0.0007591790659 0.0024195082454\n",
      "35 50 -0.0651993080974 0.000816804476804 0.00241756780786\n",
      "valid_acc 97.53\n",
      "best valid_acc 97.53\n",
      "36 0 -0.0954441726208 0.00079256064847 0.00241457628061\n",
      "36 10 -0.065805748105 0.000745725360203 0.00240866365663\n",
      "36 20 -0.0656291246414 0.000814805357718 0.00240708831571\n",
      "36 30 -0.0630030557513 0.000769361441197 0.00239911231569\n",
      "36 40 -0.0795306488872 0.000797620105092 0.00239474786519\n",
      "36 50 -0.0824849233031 0.000835353581745 0.00238931468511\n",
      "valid_acc 97.56666666666666\n",
      "best valid_acc 97.56666666666666\n",
      "37 0 -0.0607663877308 0.000839995468746 0.00238535784474\n",
      "37 10 -0.0527637600899 0.000790600122903 0.00238275474369\n",
      "37 20 -0.0946824848652 0.000787990790923 0.0023816717476\n",
      "37 30 -0.118307799101 0.000816411440093 0.00237701516921\n",
      "37 40 -0.101476542652 0.000680773269167 0.00237426792495\n",
      "37 50 -0.100584730506 0.000778281200486 0.00237337714939\n",
      "valid_acc 97.59833333333333\n",
      "best valid_acc 97.59833333333333\n",
      "38 0 -0.0621845684946 0.00079083240791 0.0023684082599\n",
      "38 10 -0.0752103850245 0.00083326647271 0.00236478463397\n",
      "38 20 -0.100619666278 0.000837856927427 0.00235893827826\n",
      "38 30 -0.107862584293 0.000942671877808 0.00235660672675\n",
      "38 40 -0.0638026669621 0.000896027608622 0.00235488907065\n",
      "38 50 -0.0858690291643 0.000883621298993 0.00234943652577\n",
      "valid_acc 97.58\n",
      "39 0 -0.0600058101118 0.000968522077767 0.00234539261305\n",
      "39 10 -0.0734886080027 0.000913649308739 0.0023390576001\n",
      "39 20 -0.0572930239141 0.000915319422961 0.00233624963926\n",
      "39 30 -0.0616451203823 0.000994602823963 0.00233112775134\n",
      "39 40 -0.0785417333245 0.000979794781211 0.00232821087425\n",
      "39 50 -0.0574326775968 0.00094064720122 0.00232605615333\n",
      "valid_acc 97.56333333333333\n",
      "40 0 -0.0695713609457 0.000932690897206 0.00232661560398\n",
      "40 10 -0.054268874228 0.000872722455717 0.00232344079548\n",
      "40 20 -0.0599458403885 0.000896832209411 0.00231882210138\n",
      "40 30 -0.0874785631895 0.000867126517523 0.00231603649142\n",
      "40 40 -0.0799371302128 0.000873482348772 0.00231032562637\n",
      "40 50 -0.0853391289711 0.000893601908588 0.00230452415148\n",
      "valid_acc 97.61833333333333\n",
      "best valid_acc 97.61833333333333\n",
      "41 0 -0.0643394291401 0.000836286535841 0.00230266624884\n",
      "41 10 -0.115371324122 0.0008470405761 0.00229952956417\n",
      "41 20 -0.0746751353145 0.000859359567072 0.00229524450477\n",
      "41 30 -0.086985707283 0.000865446901358 0.00229277676564\n",
      "41 40 -0.0628702491522 0.000837572485127 0.00228977345713\n",
      "41 50 -0.0752843692899 0.000875413309459 0.00228674837131\n",
      "valid_acc 97.67166666666667\n",
      "best valid_acc 97.67166666666667\n",
      "42 0 -0.0957732871175 0.000879192664508 0.00228505154564\n",
      "42 10 -0.0591648295522 0.0008706906624 0.00228084908391\n",
      "42 20 -0.0748505890369 0.000872591146288 0.0022777494194\n",
      "42 30 -0.0726604908705 0.000884416936264 0.00227478205856\n",
      "42 40 -0.071018435061 0.00083723190549 0.00226797754938\n",
      "42 50 -0.0601659715176 0.000901190131833 0.00226778546307\n",
      "valid_acc 97.66333333333334\n",
      "43 0 -0.0849346145988 0.000889606010607 0.0022669781507\n",
      "43 10 -0.073446765542 0.000929350923764 0.00226291293005\n",
      "43 20 -0.100777536631 0.000907866067774 0.00226104225317\n",
      "43 30 -0.0579813085496 0.00100139777685 0.00225538736785\n",
      "43 40 -0.0744867697358 0.000941130124493 0.00225079567424\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43 50 -0.0792680904269 0.000914882776134 0.00224679863005\n",
      "valid_acc 97.70166666666667\n",
      "best valid_acc 97.70166666666667\n",
      "44 0 -0.075072363019 0.000836165406018 0.00224473281394\n",
      "44 10 -0.0701611191034 0.00091151686297 0.00224041683243\n",
      "44 20 -0.069411829114 0.000828415653719 0.00223795204295\n",
      "44 30 -0.0785206630826 0.000835618069931 0.0022337878104\n",
      "44 40 -0.0848891362548 0.000808167372237 0.00223141096723\n",
      "44 50 -0.077173307538 0.0007737650749 0.00223180421032\n",
      "valid_acc 97.72166666666666\n",
      "best valid_acc 97.72166666666666\n",
      "45 0 -0.0850303843617 0.000763602986558 0.00222883808472\n",
      "45 10 -0.0666282474995 0.000779541823704 0.00222408242543\n",
      "45 20 -0.0490468293428 0.000899632008915 0.00222098279078\n",
      "45 30 -0.072115086019 0.000929868887371 0.00222007735025\n",
      "45 40 -0.0645697340369 0.000895964765108 0.00221927892535\n",
      "45 50 -0.0955443605781 0.00100496419308 0.00221507411858\n",
      "valid_acc 97.695\n",
      "46 0 -0.0653354004025 0.000943062083234 0.00221520272895\n",
      "46 10 -0.0552923940122 0.000880100645763 0.0022111725581\n",
      "46 20 -0.0951291322708 0.000895699067234 0.00220752999717\n",
      "46 30 -0.0765769332647 0.000924007919901 0.00220449644678\n",
      "46 40 -0.06977660954 0.000928857717646 0.00220394815783\n",
      "46 50 -0.0982071906328 0.000911789165754 0.00219899457196\n",
      "valid_acc 97.77666666666667\n",
      "best valid_acc 97.77666666666667\n",
      "47 0 -0.0800124108791 0.000969500264666 0.00219539986003\n",
      "47 10 -0.0866805389524 0.000905857167835 0.00219087990901\n",
      "47 20 -0.0606841035187 0.000931297936222 0.00218856412894\n",
      "47 30 -0.0674100518227 0.000878207576974 0.00218653715917\n",
      "47 40 -0.0628547519445 0.000797368786835 0.00218384369264\n",
      "47 50 -0.0465587861836 0.000828437076876 0.00218072325432\n",
      "valid_acc 97.73333333333333\n",
      "48 0 -0.0629188269377 0.000838087230216 0.00217878337965\n",
      "48 10 -0.0431539155543 0.000917386129146 0.00217687312358\n",
      "48 20 -0.0773417055607 0.000884219942525 0.00217144160865\n",
      "48 30 -0.0824057906866 0.000919294959794 0.00216839241258\n",
      "48 40 -0.0780591592193 0.000866536659489 0.00216475782071\n",
      "48 50 -0.0648985132575 0.000792370070898 0.00216066843953\n",
      "valid_acc 97.75333333333333\n",
      "49 0 -0.101374305785 0.000830550479906 0.0021550572905\n",
      "49 10 -0.0865886956453 0.000857676468529 0.00215307864863\n",
      "49 20 -0.0490629784763 0.000823530640284 0.00214570330164\n",
      "49 30 -0.0759544149041 0.000798722616035 0.0021436612146\n",
      "49 40 -0.0677886605263 0.000868629544429 0.00214116034228\n",
      "49 50 -0.0632658824325 0.00079474760017 0.00213816905852\n",
      "valid_acc 97.68166666666667\n",
      "50 0 -0.0678626000881 0.000761145321945 0.00213668222638\n",
      "50 10 -0.0522820614278 0.000824255435962 0.00213396209998\n",
      "50 20 -0.0672715529799 0.00079615758859 0.00213209846776\n",
      "50 30 -0.0797168537974 0.00086563342714 0.00213312548203\n",
      "50 40 -0.0585340596735 0.000844790869395 0.00213084808861\n",
      "50 50 -0.067994877696 0.000781536370352 0.002127033416\n",
      "valid_acc 97.69333333333333\n",
      "51 0 -0.073095664382 0.000875295964367 0.00212746361601\n",
      "51 10 -0.0834045261145 0.000908657300881 0.00212424350032\n",
      "51 20 -0.0689187571406 0.000790334340972 0.00211952347752\n",
      "51 30 -0.0632844567299 0.000822686433632 0.00211875648986\n",
      "51 40 -0.0924103632569 0.000800868503599 0.00211637274647\n",
      "51 50 -0.0689345225692 0.000880870250429 0.00211102970259\n",
      "valid_acc 97.72166666666666\n",
      "52 0 -0.0947291105986 0.000846968184172 0.00210888943584\n",
      "52 10 -0.089002572 0.000890778415982 0.00210668880783\n",
      "52 20 -0.0698742046952 0.000908033753808 0.00210191811428\n",
      "52 30 -0.0667767226696 0.000901697816572 0.0020995990245\n",
      "52 40 -0.0879523679614 0.000900569543637 0.00209621051964\n",
      "52 50 -0.0943448618054 0.000880322568763 0.00209245412307\n",
      "valid_acc 97.77333333333334\n",
      "53 0 -0.0778740420938 0.000902357576433 0.00208594457303\n",
      "53 10 -0.067453019321 0.000829116519509 0.00208325169632\n",
      "53 20 -0.0919718593359 0.000907901069295 0.00208362865449\n",
      "53 30 -0.0871060118079 0.000878529326586 0.00208362322209\n",
      "53 40 -0.0753285959363 0.000891301602449 0.00207867797141\n",
      "53 50 -0.0840071886778 0.000838101542846 0.00207686512653\n",
      "valid_acc 97.735\n",
      "54 0 -0.0857009291649 0.000856540828409 0.00207364544964\n",
      "54 10 -0.0620195008814 0.000928458922279 0.00207330554396\n",
      "54 20 -0.0659448429942 0.000909405843993 0.00207215423726\n",
      "54 30 -0.0814234316349 0.000903825783721 0.00206812466182\n",
      "54 40 -0.0444434806705 0.000819721623361 0.0020650212519\n",
      "54 50 -0.0567604489625 0.000849128183432 0.00206225902154\n",
      "valid_acc 97.78833333333333\n",
      "best valid_acc 97.78833333333333\n",
      "55 0 -0.0987149253488 0.000856154825741 0.0020613320909\n",
      "55 10 -0.0786830112338 0.000815956091566 0.00205868809973\n",
      "55 20 -0.0703766494989 0.000779840919311 0.00205876572591\n",
      "55 30 -0.0728942602873 0.000749020169418 0.0020547745199\n",
      "55 40 -0.101957194507 0.000769530085517 0.00205076530821\n",
      "55 50 -0.0630824640393 0.000760654049899 0.0020477216454\n",
      "valid_acc 97.74666666666667\n",
      "56 0 -0.069021217525 0.000753309303124 0.00204565541595\n",
      "56 10 -0.0624925121665 0.000709239778572 0.00204413411096\n",
      "56 20 -0.0595703162253 0.000693382208807 0.00204564040481\n",
      "56 30 -0.0635353922844 0.000749134687351 0.00204123363125\n",
      "56 40 -0.0784621685743 0.000706445933329 0.00204026239134\n",
      "56 50 -0.06208634004 0.000709207274993 0.00203677206585\n",
      "valid_acc 97.715\n",
      "57 0 -0.0858890041709 0.000769726788553 0.00203175496427\n",
      "57 10 -0.0609682016075 0.000769269663244 0.00203058020893\n",
      "57 20 -0.0663688257337 0.000747668025769 0.00202697853082\n",
      "57 30 -0.0469357594848 0.000757533567352 0.0020271250007\n",
      "57 40 -0.0602676570415 0.000862774773423 0.00202782270449\n",
      "57 50 -0.0689224377275 0.000846492202034 0.00202782248639\n",
      "valid_acc 97.78999999999999\n",
      "best valid_acc 97.78999999999999\n",
      "58 0 -0.0611949935555 0.000885035218408 0.00202482058783\n",
      "58 10 -0.0849110111594 0.000898055465668 0.00202173624298\n",
      "58 20 -0.0478942878544 0.000874202478611 0.00201983792082\n",
      "58 30 -0.0571486763656 0.000896188156921 0.0020179430264\n",
      "58 40 -0.0852044001222 0.000951445551051 0.00201293921083\n",
      "58 50 -0.0748377889395 0.00102293218283 0.00201123761129\n",
      "valid_acc 97.75\n",
      "59 0 -0.0674690753222 0.0010037334318 0.00200776489855\n",
      "59 10 -0.087249211967 0.00102710396061 0.00200430310068\n",
      "59 20 -0.0669419541955 0.000951406743521 0.0020036836979\n",
      "59 30 -0.0667329728603 0.0010097565621 0.00200335310412\n",
      "59 40 -0.0710822492838 0.00102152032597 0.00200211659432\n",
      "59 50 -0.0790116414428 0.00102052195583 0.00199911200223\n",
      "valid_acc 97.82333333333332\n",
      "best valid_acc 97.82333333333332\n",
      "60 0 -0.0910212025046 0.00101500369249 0.00199448847521\n",
      "60 10 -0.0518956147134 0.00100542441358 0.00199319247087\n",
      "60 20 -0.072878330946 0.00110327888663 0.00199109674301\n",
      "60 30 -0.0510129742324 0.00101671460227 0.00198803325336\n",
      "60 40 -0.0758544504642 0.00102032695955 0.00198683802088\n",
      "60 50 -0.0688111558557 0.00104289662424 0.00198508503145\n",
      "valid_acc 97.82333333333332\n",
      "best valid_acc 97.82333333333332\n",
      "61 0 -0.0560882352293 0.000999009513045 0.00198380728796\n",
      "61 10 -0.0673373937607 0.00100933292546 0.00198094698713\n",
      "61 20 -0.0772756487131 0.000986103783757 0.00197804212473\n",
      "61 30 -0.0594056621194 0.00100358338816 0.00197331258198\n",
      "61 40 -0.0690764039755 0.000972293742727 0.0019727209229\n",
      "61 50 -0.054340865463 0.00100330288366 0.00197008675206\n",
      "valid_acc 97.84333333333333\n",
      "best valid_acc 97.84333333333333\n",
      "62 0 -0.0633680596948 0.000976205642001 0.00196804942402\n",
      "62 10 -0.0591195188463 0.000918664705975 0.00196496806246\n",
      "62 20 -0.0359679460526 0.000993846055784 0.00196407507932\n",
      "62 30 -0.0446812622249 0.000927292736976 0.00195936270654\n",
      "62 40 -0.05496410653 0.000933241754459 0.00195715008153\n",
      "62 50 -0.0724743083119 0.000908667814018 0.00195426035589\n",
      "valid_acc 97.85833333333333\n",
      "best valid_acc 97.85833333333333\n",
      "63 0 -0.0671240612864 0.000953063001933 0.00195204969351\n",
      "63 10 -0.0597227215767 0.000960898226489 0.00195220299821\n",
      "63 20 -0.0615901425481 0.000973407616041 0.00195049162166\n",
      "63 30 -0.0698002576828 0.000926711166239 0.00194397800675\n",
      "63 40 -0.0635195076466 0.000912999629477 0.00194174068335\n",
      "63 50 -0.0774204432964 0.000983576806572 0.00194043120702\n",
      "valid_acc 97.86166666666666\n",
      "best valid_acc 97.86166666666666\n",
      "64 0 -0.0495466180146 0.00100262786998 0.00194164018025\n",
      "64 10 -0.0676609799266 0.00102323744005 0.00193909963806\n",
      "64 20 -0.0621678121388 0.00100605081984 0.00193861436018\n",
      "64 30 -0.0625037252903 0.00101113861253 0.00193618670305\n",
      "64 40 -0.0771643295884 0.00106717846238 0.00193342329899\n",
      "64 50 -0.0650186985731 0.00107367770329 0.00192781252123\n",
      "valid_acc 97.84833333333334\n",
      "65 0 -0.0661346539855 0.00106718802888 0.00192707358482\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65 10 -0.0551601871848 0.00101899955405 0.00192477046671\n",
      "65 20 -0.0618918389082 0.00103471800953 0.00192364485393\n",
      "65 30 -0.0420793183148 0.0010320759224 0.00192650656589\n",
      "65 40 -0.0882163867354 0.00110078139627 0.0019249035331\n",
      "65 50 -0.0591966845095 0.00108420629361 0.0019235407947\n",
      "valid_acc 97.89666666666666\n",
      "best valid_acc 97.89666666666666\n",
      "66 0 -0.0564950481057 0.00106584435556 0.00192256792892\n",
      "66 10 -0.0769885703921 0.00112152037154 0.00191855229838\n",
      "66 20 -0.0795011892915 0.00102180580491 0.00191808438065\n",
      "66 30 -0.0635740086436 0.00101875772466 0.0019152974685\n",
      "66 40 -0.0733740255237 0.00108151591709 0.0019149526173\n",
      "66 50 -0.076630756259 0.00105306325432 0.00191427066544\n",
      "valid_acc 97.84833333333334\n",
      "67 0 -0.0518630295992 0.0010034458747 0.00191294091986\n",
      "67 10 -0.0835065096617 0.00104453128774 0.00191106540714\n",
      "67 20 -0.0626259073615 0.00106560312265 0.00190890213556\n",
      "67 30 -0.060091484338 0.00106153885739 0.00190635538456\n",
      "67 40 -0.0508683472872 0.0010358376257 0.00190357294224\n",
      "67 50 -0.0604690462351 0.00101629898361 0.00190147783448\n",
      "valid_acc 97.88833333333334\n",
      "68 0 -0.0693193003535 0.00101660363342 0.00189670851617\n",
      "68 10 -0.0630562379956 0.00104230949721 0.00189372338917\n",
      "68 20 -0.0738985612988 0.00105517302021 0.00189158314398\n",
      "68 30 -0.0659157857299 0.000996692561452 0.00188911174099\n",
      "68 40 -0.0726225823164 0.000981353704603 0.00188713296189\n",
      "68 50 -0.0622058883309 0.00100055452957 0.00188333734798\n",
      "valid_acc 97.89333333333333\n",
      "69 0 -0.0741473734379 0.000968096960082 0.00188008595057\n",
      "69 10 -0.050063200295 0.000979291871519 0.00187964946478\n",
      "69 20 -0.0814615264535 0.00105375239017 0.00187554782369\n",
      "69 30 -0.108646273613 0.00102819561339 0.00187677630563\n",
      "69 40 -0.0606661364436 0.00101180597282 0.00187338241494\n",
      "69 50 -0.0642534643412 0.0010643685851 0.0018716999355\n",
      "valid_acc 97.92166666666667\n",
      "best valid_acc 97.92166666666667\n",
      "70 0 -0.0691145434976 0.00101517434421 0.00186827229871\n",
      "70 10 -0.0565928965807 0.000971208855856 0.00186557229432\n",
      "70 20 -0.0684570521116 0.000979591204969 0.00186365380903\n",
      "70 30 -0.0982895046473 0.00099924236221 0.0018619297986\n",
      "70 40 -0.0993212983012 0.000973685463751 0.0018606627636\n",
      "70 50 -0.0458309948444 0.000989820432115 0.00186053817241\n",
      "valid_acc 97.98166666666667\n",
      "best valid_acc 97.98166666666667\n",
      "71 0 -0.0626209303737 0.000999985934219 0.0018571738446\n",
      "71 10 -0.0564146377146 0.000956198808335 0.00185540444745\n",
      "71 20 -0.0550193637609 0.00100790560199 0.00185368857616\n",
      "71 30 -0.0553949251771 0.000917544114546 0.00185231370564\n",
      "71 40 -0.0620043724775 0.000940707697208 0.00185118348704\n",
      "71 50 -0.061731684953 0.000952225451051 0.00184922916319\n",
      "valid_acc 97.95666666666666\n",
      "72 0 -0.0688015818596 0.00102107433746 0.00184706637518\n",
      "72 10 -0.0749062523246 0.000995824703581 0.00184623260191\n",
      "72 20 -0.064060613513 0.000946743056381 0.00184450296503\n",
      "72 30 -0.0569655150175 0.000895229588879 0.00184098690569\n",
      "72 40 -0.0603754743934 0.000932145607884 0.0018379617612\n",
      "72 50 -0.0657326802611 0.000949158298337 0.0018362937226\n",
      "valid_acc 97.92833333333333\n",
      "73 0 -0.0471868515015 0.000877304173614 0.00183541641576\n",
      "73 10 -0.0671922191978 0.000901401452307 0.00183144980973\n",
      "73 20 -0.06249486655 0.000949970366394 0.00182961730268\n",
      "73 30 -0.0384009107947 0.000949133956041 0.00182977357851\n",
      "73 40 -0.050106190145 0.000924707158082 0.00182954514275\n",
      "73 50 -0.0801233127713 0.000926692095372 0.00182658010842\n",
      "valid_acc 97.98166666666667\n",
      "best valid_acc 97.98166666666667\n",
      "74 0 -0.059365209192 0.000903306472799 0.00182578584947\n",
      "74 10 -0.0657164901495 0.000889497767926 0.0018261823317\n",
      "74 20 -0.0532665736973 0.000899325844657 0.00182307273309\n",
      "74 30 -0.0509440638125 0.000978435608166 0.00182232033812\n",
      "74 40 -0.0826149061322 0.000931610307634 0.00182038734439\n",
      "74 50 -0.0431626848876 0.000977117352538 0.00181840909019\n",
      "valid_acc 97.95166666666667\n",
      "75 0 -0.0729602724314 0.000902438625089 0.00181338155329\n",
      "75 10 -0.0467497818172 0.000929394215198 0.0018147714233\n",
      "75 20 -0.0727706998587 0.000962801899447 0.00181333228156\n",
      "75 30 -0.0447062402964 0.000935278785154 0.00181249877618\n",
      "75 40 -0.0632866024971 0.00103295345002 0.00181044019696\n",
      "75 50 -0.072931535542 0.000965590969969 0.00180819764028\n",
      "valid_acc 97.96666666666667\n",
      "76 0 -0.0526820644736 0.0009516067016 0.00180644508957\n",
      "76 10 -0.082777634263 0.000959421601088 0.00180535047314\n",
      "76 20 -0.0805727988482 0.000942459155479 0.00180503977443\n",
      "76 30 -0.0653473436832 0.00100681448685 0.00180575271963\n",
      "76 40 -0.0559695847332 0.000973830634133 0.00180231051405\n",
      "76 50 -0.0683535784483 0.000909687247042 0.00179972254124\n",
      "valid_acc 97.975\n",
      "77 0 -0.0498190224171 0.000979965664505 0.00179777095544\n",
      "77 10 -0.0610929131508 0.00096301215366 0.00179613309592\n",
      "77 20 -0.0799521580338 0.00102335008127 0.00179631773997\n",
      "77 30 -0.0434830784798 0.000989922592751 0.00179470494954\n",
      "77 40 -0.055815923959 0.00100585994729 0.00179431511675\n",
      "77 50 -0.0402360931039 0.000967366944467 0.00179339993816\n",
      "valid_acc 97.965\n",
      "78 0 -0.0590526498854 0.000990530157938 0.00179362742356\n",
      "78 10 -0.0663658082485 0.000994462996298 0.00179189297628\n",
      "78 20 -0.0733836218715 0.000944985878873 0.00179003456417\n",
      "78 30 -0.0747561976314 0.00097404427858 0.0017862875363\n",
      "78 40 -0.0671813189983 0.00100929606954 0.00178515809564\n",
      "78 50 -0.0545134581625 0.000918487059186 0.00178395855694\n",
      "valid_acc 98.01833333333333\n",
      "best valid_acc 98.01833333333333\n",
      "79 0 -0.0619371570647 0.000972532308392 0.00177941633878\n",
      "79 10 -0.0534271709621 0.000952542720723 0.0017780212145\n",
      "79 20 -0.0538930594921 0.000945694619845 0.00177513815335\n",
      "79 30 -0.0443243049085 0.000947463568221 0.00177368075108\n",
      "79 40 -0.0575834736228 0.000961992397586 0.00177143531163\n",
      "79 50 -0.05028777197 0.0010078966998 0.00176927926577\n",
      "valid_acc 98.015\n",
      "80 0 -0.0776270180941 0.000960969906715 0.00177062601733\n",
      "80 10 -0.0744371339679 0.00102944867541 0.00177147518945\n",
      "80 20 -0.0584052801132 0.00101377393142 0.00176808535669\n",
      "80 30 -0.0462822690606 0.000981550479651 0.00176576324649\n",
      "80 40 -0.0613155551255 0.000937072908695 0.00176403585575\n",
      "80 50 -0.0720825791359 0.0009312935227 0.00176569940175\n",
      "valid_acc 98.0\n",
      "81 0 -0.0647581294179 0.000982442571786 0.00176412863357\n",
      "81 10 -0.068183593452 0.00101487987668 0.00176157136219\n",
      "81 20 -0.0478085875511 0.00100663779936 0.00176031765849\n",
      "81 30 -0.0649609491229 0.000937548088305 0.00176057327993\n",
      "81 40 -0.0642245039344 0.000995455546908 0.0017599811906\n",
      "81 50 -0.0527836754918 0.000922384927303 0.00175805851225\n",
      "valid_acc 98.0\n",
      "82 0 -0.0514001213014 0.00100736678909 0.00175499773809\n",
      "82 10 -0.0521895289421 0.000964643128355 0.00175268466193\n",
      "82 20 -0.058268725872 0.000910348971758 0.00174950106207\n",
      "82 30 -0.0521750971675 0.000926999008177 0.00175046228571\n",
      "82 40 -0.064679376781 0.000937883366396 0.00174891317338\n",
      "82 50 -0.0656731277704 0.000908316520225 0.00174727999948\n",
      "valid_acc 98.04833333333333\n",
      "best valid_acc 98.04833333333333\n",
      "83 0 -0.0838786885142 0.000914958729156 0.00174539845632\n",
      "83 10 -0.0664583295584 0.000918692186072 0.00174435641752\n",
      "83 20 -0.0600226633251 0.000894070666751 0.00174467914573\n",
      "83 30 -0.0797089189291 0.000922798609039 0.00174534798336\n",
      "83 40 -0.0969624891877 0.000933969500208 0.00174655419502\n",
      "83 50 -0.0451970174909 0.000964256620874 0.00174712489417\n",
      "valid_acc 98.03333333333333\n",
      "84 0 -0.0628429725766 0.000906593476775 0.00174676277581\n",
      "84 10 -0.0485133603215 0.000871292744007 0.00174742944489\n",
      "84 20 -0.0648955628276 0.000889383002405 0.00174664141772\n",
      "84 30 -0.0895208269358 0.000928599405486 0.00174550565728\n",
      "84 40 -0.0573744364083 0.000884608366498 0.00174423469309\n",
      "84 50 -0.0560118369758 0.000862420442187 0.00174160425924\n",
      "valid_acc 97.995\n",
      "85 0 -0.0701581016183 0.000895487954246 0.00174033662407\n",
      "85 10 -0.0646138563752 0.00087120557794 0.00173830295085\n",
      "85 20 -0.0678824037313 0.000996864791671 0.00173881112248\n",
      "85 30 -0.0431652404368 0.000931986451195 0.00173757501702\n",
      "85 40 -0.0843843147159 0.00094939948461 0.00173682030342\n",
      "85 50 -0.0404578112066 0.000912925803245 0.00173456899095\n",
      "valid_acc 98.02333333333333\n",
      "86 0 -0.0423459075391 0.000877713544337 0.00173333125426\n",
      "86 10 -0.046411767602 0.000946733993116 0.00173110937245\n",
      "86 20 -0.058477897197 0.000904848223163 0.00172704024784\n",
      "86 30 -0.10389482975 0.00089759276746 0.00172473377581\n",
      "86 40 -0.0549925491214 0.000903093836781 0.00172578691671\n",
      "86 50 -0.0627077296376 0.00091837987651 0.00172451113951\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "valid_acc 98.02\n",
      "87 0 -0.0777481198311 0.000940643831532 0.00172023912799\n",
      "87 10 -0.096484683454 0.000939182971627 0.00171984165381\n",
      "87 20 -0.0778428837657 0.000920028409411 0.00171694432099\n",
      "87 30 -0.0822711363435 0.000915325942824 0.00171623849054\n",
      "87 40 -0.0805753022432 0.000915709264529 0.00171382652637\n",
      "87 50 -0.0416690334678 0.000938822345066 0.00171345517599\n",
      "valid_acc 98.01833333333333\n",
      "88 0 -0.0676549896598 0.000937524115906 0.00171290754584\n",
      "88 10 -0.0408354438841 0.000911403998275 0.00171226837614\n",
      "88 20 -0.0545636191964 0.000942955804327 0.00170993460011\n",
      "88 30 -0.0477593317628 0.000894997811992 0.00170701387578\n",
      "88 40 -0.0569612495601 0.000878442887661 0.00170510410988\n",
      "88 50 -0.0681399330497 0.000874688110441 0.0017042411075\n",
      "valid_acc 98.00666666666666\n",
      "89 0 -0.0612490214407 0.000889218206416 0.00170261873047\n",
      "89 10 -0.0507856570184 0.00088574976998 0.00170065127694\n",
      "89 20 -0.0541860610247 0.000871785942116 0.00170021476813\n",
      "89 30 -0.0447176061571 0.000909938222834 0.00169707009964\n",
      "89 40 -0.0458347611129 0.00092301706496 0.00169299578221\n",
      "89 50 -0.0636148974299 0.000851430611088 0.00169206500439\n",
      "valid_acc 98.08666666666667\n",
      "best valid_acc 98.08666666666667\n",
      "90 0 -0.0660084709525 0.000921558046932 0.00169153325554\n",
      "90 10 -0.0618583112955 0.000881403593776 0.00168955713774\n",
      "90 20 -0.0545303337276 0.000901777538236 0.00168842587373\n",
      "90 30 -0.0639667436481 0.000951962607543 0.00168699224021\n",
      "90 40 -0.0348572395742 0.000936726115475 0.00168632142912\n",
      "90 50 -0.0405433252454 0.000944326895625 0.00168519326521\n",
      "valid_acc 98.10333333333332\n",
      "best valid_acc 98.10333333333332\n",
      "91 0 -0.0607943497598 0.000974765084934 0.00168442629289\n",
      "91 10 -0.0637477859855 0.000956135635508 0.00168370842464\n",
      "91 20 -0.0663855299354 0.00100561083925 0.00168058789526\n",
      "91 30 -0.073567956686 0.000930612258977 0.00167835291822\n",
      "91 40 -0.0495889149606 0.000975089496373 0.00167704236543\n",
      "91 50 -0.0634032264352 0.000897673296722 0.00167563139031\n",
      "valid_acc 98.10333333333332\n",
      "best valid_acc 98.10333333333332\n",
      "92 0 -0.0770059674978 0.00101898297007 0.00167440411647\n",
      "92 10 -0.0449941046536 0.000973585621933 0.00167455134229\n",
      "92 20 -0.0416094996035 0.00100810299373 0.00167437141626\n",
      "92 30 -0.055460549891 0.00091151423375 0.00167238394314\n",
      "92 40 -0.0666783228517 0.000925751750735 0.00166954095626\n",
      "92 50 -0.0648067891598 0.000887368792104 0.00166844608163\n",
      "valid_acc 98.03500000000001\n",
      "93 0 -0.059121824801 0.000943354535481 0.0016681261966\n",
      "93 10 -0.0486593022943 0.000963999354737 0.00166454801813\n",
      "93 20 -0.0615062490106 0.000995774691432 0.00166323815623\n",
      "93 30 -0.0719784721732 0.00088480622463 0.0016607843508\n",
      "93 40 -0.0633070170879 0.000880982762492 0.00166017678587\n",
      "93 50 -0.0473600663245 0.00087139217916 0.00165871362471\n",
      "valid_acc 98.05333333333334\n",
      "94 0 -0.0558058917522 0.000863882707377 0.00165706558582\n",
      "94 10 -0.0692510381341 0.000898215942551 0.00165462195116\n",
      "94 20 -0.0448820739985 0.000855823130037 0.00165289994665\n",
      "94 30 -0.0726057216525 0.000866396001084 0.0016508496918\n",
      "94 40 -0.034600533545 0.000787551957182 0.00164938112997\n",
      "94 50 -0.0538180284202 0.000839535921916 0.00164716691347\n",
      "valid_acc 98.07666666666667\n",
      "95 0 -0.0538425520062 0.0008741031245 0.00164508105912\n",
      "95 10 -0.0491554550827 0.00090409087299 0.00164350509018\n",
      "95 20 -0.0438895300031 0.000898422823661 0.00164419675659\n",
      "95 30 -0.0745923370123 0.000870729398874 0.00164356172832\n",
      "95 40 -0.0452623851597 0.000872808257803 0.00164022660364\n",
      "95 50 -0.059683624655 0.000939359003259 0.00164084432648\n",
      "valid_acc 98.06\n",
      "96 0 -0.0755808353424 0.000836045524779 0.00164039401686\n",
      "96 10 -0.06174659729 0.00095092937899 0.00163848364321\n",
      "96 20 -0.0836167931557 0.00090833066946 0.00163589226461\n",
      "96 30 -0.0566252246499 0.000957253876085 0.00163391742049\n",
      "96 40 -0.0417882986367 0.000955260300709 0.00163066287204\n",
      "96 50 -0.0791959315538 0.000935498456929 0.00162912949859\n",
      "valid_acc 98.05\n",
      "97 0 -0.0589312836528 0.000945140959759 0.00162887712248\n",
      "97 10 -0.074634462595 0.000936003254369 0.00162778222034\n",
      "97 20 -0.056673400104 0.000926761493505 0.00162711278229\n",
      "97 30 -0.0364587008953 0.000910534199966 0.00162648251611\n",
      "97 40 -0.0607174374163 0.00088501744524 0.00162524100888\n",
      "97 50 -0.0550161153078 0.0008762307121 0.00162371611231\n",
      "valid_acc 98.08666666666667\n",
      "98 0 -0.0632099062204 0.000914587891342 0.00162153061651\n",
      "98 10 -0.0742261111736 0.000872568649323 0.00161860264098\n",
      "98 20 -0.0768845528364 0.000935816042083 0.00161842091807\n",
      "98 30 -0.0535627640784 0.000939518982641 0.00161422251754\n",
      "98 40 -0.0572390146554 0.000879787660338 0.0016138235034\n",
      "98 50 -0.0428619682789 0.00092433182125 0.0016140151632\n",
      "valid_acc 98.065\n",
      "99 0 -0.0703194588423 0.00089938422331 0.00161146824315\n",
      "99 10 -0.0652059391141 0.000951177154447 0.00161021404659\n",
      "99 20 -0.065917827189 0.000957591202309 0.00160866589771\n",
      "99 30 -0.0578117743134 0.000926504581393 0.00160661566655\n",
      "99 40 -0.0532854795456 0.000915314053127 0.00160533523677\n",
      "99 50 -0.0824108272791 0.000899143510971 0.00160353384126\n",
      "valid_acc 98.04666666666667\n",
      "100 0 -0.0374194942415 0.000943643211349 0.00160421695355\n",
      "100 10 -0.0638034194708 0.000932166051768 0.00160419947393\n",
      "100 20 -0.0632742866874 0.000933659380102 0.00160287909453\n",
      "100 30 -0.0607291199267 0.000972646633426 0.00160233008572\n",
      "100 40 -0.0705016329885 0.000957030289252 0.00160158232027\n",
      "100 50 -0.0600859485567 0.000982164433307 0.00159907717252\n",
      "valid_acc 98.12166666666667\n",
      "best valid_acc 98.12166666666667\n",
      "101 0 -0.0443351753056 0.000974826506787 0.00160068225104\n",
      "101 10 -0.060005620122 0.000988642900007 0.00159945283342\n",
      "101 20 -0.0935455262661 0.00096610461533 0.00159926769796\n",
      "101 30 -0.0458681434393 0.000892023088298 0.00159881554352\n",
      "101 40 -0.04735988006 0.00088391478872 0.00159652967673\n",
      "101 50 -0.0473527312279 0.000945025692916 0.00159567677595\n",
      "valid_acc 98.07000000000001\n",
      "102 0 -0.0373088605702 0.000937668944207 0.00159374214847\n",
      "102 10 -0.045034237206 0.000962622990542 0.00159135224173\n",
      "102 20 -0.0685488954186 0.000930440870135 0.00159198303307\n",
      "102 30 -0.0571148730814 0.000913053730203 0.00158940855778\n",
      "102 40 -0.0495847724378 0.000930048419717 0.00158885812664\n",
      "102 50 -0.0779059156775 0.000921291757975 0.00158866409918\n",
      "valid_acc 98.13166666666666\n",
      "best valid_acc 98.13166666666666\n",
      "103 0 -0.0539617985487 0.000966096811555 0.00158847525873\n",
      "103 10 -0.0465403944254 0.000954626285512 0.0015873842874\n",
      "103 20 -0.033874232322 0.000936470495717 0.0015870634701\n",
      "103 30 -0.0509653612971 0.000979559967499 0.0015857430639\n",
      "103 40 -0.0502147376537 0.00102185128697 0.00158466424907\n",
      "103 50 -0.060563467443 0.000988537627142 0.00158408384195\n",
      "valid_acc 98.12833333333333\n",
      "104 0 -0.0689146891236 0.000967509655781 0.0015838555426\n",
      "104 10 -0.059033934027 0.0010133122094 0.00158289113454\n",
      "104 20 -0.0602011159062 0.000932561791046 0.00158191568539\n",
      "104 30 -0.0590440817177 0.00090179248662 0.0015830784619\n",
      "104 40 -0.0639892667532 0.000924105426336 0.00158245519614\n",
      "104 50 -0.0586908273399 0.000967169249816 0.00158189533619\n",
      "valid_acc 98.155\n",
      "best valid_acc 98.155\n",
      "105 0 -0.0634963288903 0.000948757123308 0.00158130930648\n",
      "105 10 -0.0621558055282 0.000947644350446 0.00157985781767\n",
      "105 20 -0.0477008372545 0.00100879558144 0.00157843691379\n",
      "105 30 -0.0576192960143 0.000946701190997 0.00157749414667\n",
      "105 40 -0.0578346736729 0.000939211965668 0.00157495112829\n",
      "105 50 -0.0593102499843 0.000983349076004 0.00157253976147\n",
      "valid_acc 98.12333333333333\n",
      "106 0 -0.0609729588032 0.000930499973499 0.00157029512142\n",
      "106 10 -0.0444496907294 0.000932096016439 0.0015680508966\n",
      "106 20 -0.0696717351675 0.0009448194553 0.00156717437296\n",
      "106 30 -0.071718968451 0.000975665925611 0.00156436688736\n",
      "106 40 -0.0647897273302 0.000906192750403 0.00156321049155\n",
      "106 50 -0.0437019057572 0.000890563637133 0.00156139103111\n",
      "valid_acc 98.14166666666667\n",
      "107 0 -0.0544052012265 0.000953106824507 0.0015606950638\n",
      "107 10 -0.0589485131204 0.00101440402095 0.0015590127657\n",
      "107 20 -0.0393206998706 0.000986785624154 0.00155917497449\n",
      "107 30 -0.061154525727 0.000985032126402 0.00155703625588\n",
      "107 40 -0.0663799494505 0.00101761661695 0.00155765997905\n",
      "107 50 -0.0469104535878 0.000988045347472 0.00155614231467\n",
      "valid_acc 98.14500000000001\n",
      "108 0 -0.0540928281844 0.00101316544175 0.00155530399704\n",
      "108 10 -0.0496025234461 0.00101411345008 0.00155320044711\n",
      "108 20 -0.0469730384648 0.000990506143448 0.00155120635155\n",
      "108 30 -0.0567871257663 0.000998663262567 0.00155036795633\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "108 40 -0.0556872114539 0.00106142434395 0.00155019819748\n",
      "108 50 -0.0595418028533 0.0010605068868 0.00154758505204\n",
      "valid_acc 98.11333333333333\n",
      "109 0 -0.0369131043553 0.00103928407782 0.00154645192774\n",
      "109 10 -0.0623419955373 0.00102584045853 0.00154575932014\n",
      "109 20 -0.0577274262905 0.00102343489173 0.00154530877328\n",
      "109 30 -0.0563415847719 0.00105209152399 0.00154457958783\n",
      "109 40 -0.0651528611779 0.00101600109503 0.00154522319947\n",
      "109 50 -0.0583098307252 0.0010535386947 0.00154579014627\n",
      "valid_acc 98.16\n",
      "best valid_acc 98.16\n",
      "110 0 -0.0586435981095 0.00106064693875 0.00154446707534\n",
      "110 10 -0.0646004751325 0.00103500394315 0.00154264059704\n",
      "110 20 -0.0640068277717 0.00107025638743 0.00153893721339\n",
      "110 30 -0.0749094486237 0.00105418578371 0.00153685666784\n",
      "110 40 -0.0879976674914 0.00106998675532 0.00153593208991\n",
      "110 50 -0.045017555356 0.00105410679326 0.00153495861022\n",
      "valid_acc 98.16833333333334\n",
      "best valid_acc 98.16833333333334\n",
      "111 0 -0.0624889619648 0.00112438855222 0.00153423246139\n",
      "111 10 -0.0582691058517 0.00114247367602 0.0015325372117\n",
      "111 20 -0.101775594056 0.00109065991406 0.00153182736173\n",
      "111 30 -0.0499654300511 0.00107194671937 0.001530847994\n",
      "111 40 -0.0624065697193 0.00103335267155 0.00153097306005\n",
      "111 50 -0.0604842863977 0.00103115630016 0.00152939298602\n",
      "valid_acc 98.12166666666667\n",
      "112 0 -0.0691769570112 0.00104067555391 0.00152796926324\n",
      "112 10 -0.0416245907545 0.00100892407211 0.00152800939396\n",
      "112 20 -0.0502525791526 0.00108781180832 0.00152688680058\n",
      "112 30 -0.0468201935291 0.00102004414813 0.00152779241106\n",
      "112 40 -0.0464553572237 0.0010245530608 0.00152704286817\n",
      "112 50 -0.0565340444446 0.00103038862792 0.00152615690073\n",
      "valid_acc 98.16333333333334\n",
      "113 0 -0.0543060451746 0.00104709395939 0.00152682126843\n",
      "113 10 -0.0525717064738 0.000969947399929 0.00152671996938\n",
      "113 20 -0.0530771054327 0.00101928580732 0.00152562260185\n",
      "113 30 -0.0506352484226 0.00106714241063 0.00152413941444\n",
      "113 40 -0.0595132783055 0.00105091350528 0.0015238644177\n",
      "113 50 -0.0632568672299 0.00102683558034 0.00152528381237\n",
      "valid_acc 98.12833333333333\n",
      "114 0 -0.0680893883109 0.000987880094961 0.00152467931743\n",
      "114 10 -0.0703449249268 0.00101972990157 0.00152476348923\n",
      "114 20 -0.0571644864976 0.00100874802456 0.00152455605715\n",
      "114 30 -0.0599075295031 0.00102509878929 0.00152324297486\n",
      "114 40 -0.0345547497272 0.00101697031351 0.00152316307194\n",
      "114 50 -0.0684372782707 0.00102776397715 0.00152243817323\n",
      "valid_acc 98.13\n",
      "115 0 -0.0690058842301 0.00102392767648 0.00152139690686\n",
      "115 10 -0.0474396906793 0.00101443165377 0.00152109257334\n",
      "115 20 -0.0628973916173 0.00107201615627 0.00152033184855\n",
      "115 30 -0.0639133676887 0.000964473288782 0.00151870686824\n",
      "115 40 -0.0572389923036 0.00102174093072 0.00151800608645\n",
      "115 50 -0.0614593848586 0.00101280861426 0.00151524827413\n",
      "valid_acc 98.2\n",
      "best valid_acc 98.2\n",
      "116 0 -0.0581579320133 0.00100104118444 0.00151340367881\n",
      "116 10 -0.0802026987076 0.00103891899294 0.00151097791263\n",
      "116 20 -0.0604534894228 0.00103886155826 0.00151045915794\n",
      "116 30 -0.066201262176 0.00100302479373 0.00151091744939\n",
      "116 40 -0.0815343409777 0.00101906283816 0.00150808330511\n",
      "116 50 -0.0539462044835 0.00102000386175 0.00150816803611\n",
      "valid_acc 98.16666666666667\n",
      "117 0 -0.0512875393033 0.000978362525419 0.00150777834753\n",
      "117 10 -0.0519607886672 0.000934103631102 0.00150736754972\n",
      "117 20 -0.0487578101456 0.000980051360828 0.00150543392035\n",
      "117 30 -0.0531307160854 0.00102321998587 0.00150507690794\n",
      "117 40 -0.085332326591 0.000954128931401 0.00150555682577\n",
      "117 50 -0.0644780471921 0.00098238431668 0.00150506608903\n",
      "valid_acc 98.17833333333333\n",
      "118 0 -0.063048876822 0.000952068797555 0.00150404761532\n",
      "118 10 -0.0316310189664 0.00095062993611 0.00150276915764\n",
      "118 20 -0.0392772592604 0.00099308773405 0.00150194679995\n",
      "118 30 -0.0479938015342 0.000968682710845 0.00150075343266\n",
      "118 40 -0.041551195085 0.000963892795904 0.00149973506629\n",
      "118 50 -0.0513466969132 0.000935406499179 0.00149880869754\n",
      "valid_acc 98.15333333333334\n",
      "119 0 -0.0502860657871 0.000927222758842 0.00149762794898\n",
      "119 10 -0.0554524622858 0.000960981000299 0.00149645514363\n",
      "119 20 -0.0658994615078 0.000979220105569 0.00149632875787\n",
      "119 30 -0.059140868485 0.000983635459612 0.00149544639599\n",
      "119 40 -0.0651154145598 0.000954578306462 0.00149557021619\n",
      "119 50 -0.0692130476236 0.000949675248643 0.0014954193003\n",
      "valid_acc 98.17166666666667\n",
      "120 0 -0.0893390253186 0.000912436035408 0.00149583346317\n",
      "120 10 -0.0642639771104 0.000939297819805 0.00149447955232\n",
      "120 20 -0.0516867190599 0.000921019396291 0.00149332213625\n",
      "120 30 -0.0453649386764 0.000921559160355 0.00149190235522\n",
      "120 40 -0.065965436399 0.000979420528328 0.001490983616\n",
      "120 50 -0.0247657541186 0.000931640519686 0.00149067512034\n",
      "valid_acc 98.16833333333334\n",
      "121 0 -0.0467200241983 0.00090365305088 0.00148966742698\n",
      "121 10 -0.0571050420403 0.000897662767822 0.00148880856117\n",
      "121 20 -0.0447224751115 0.0009038937712 0.00148865258341\n",
      "121 30 -0.0576079562306 0.000873654707494 0.00148756924606\n",
      "121 40 -0.0404568761587 0.000890483940398 0.00148697836725\n",
      "121 50 -0.0507467463613 0.000893018997223 0.001485283747\n",
      "valid_acc 98.18333333333334\n",
      "122 0 -0.0531623065472 0.00085772484669 0.00148444917338\n",
      "122 10 -0.0677298232913 0.000885101314443 0.00148409181438\n",
      "122 20 -0.0521187409759 0.000856797356528 0.00148249163678\n",
      "122 30 -0.0618718974292 0.000872914139028 0.0014827880399\n",
      "122 40 -0.065201677382 0.000891500727546 0.00148213066897\n",
      "122 50 -0.0501512400806 0.000885959062056 0.00148241868785\n",
      "valid_acc 98.22833333333332\n",
      "best valid_acc 98.22833333333332\n",
      "123 0 -0.0240825116634 0.000869349409564 0.00148274751009\n",
      "123 10 -0.0663093328476 0.000925407478812 0.00148040801162\n",
      "123 20 -0.0364975295961 0.00089923188516 0.00148151013119\n",
      "123 30 -0.0699323788285 0.000857129047362 0.00148060975395\n",
      "123 40 -0.0736439749599 0.000862711474381 0.00148163003743\n",
      "123 50 -0.0672136470675 0.000903238905019 0.0014816335458\n",
      "valid_acc 98.19\n",
      "124 0 -0.0481516383588 0.00091252014758 0.00148155815955\n",
      "124 10 -0.0558201000094 0.000927732636823 0.00148151513316\n",
      "124 20 -0.0509186796844 0.000895001560935 0.00148075714188\n",
      "124 30 -0.0649056956172 0.00088489707249 0.00148137527657\n",
      "124 40 -0.0577910803258 0.000894731217049 0.00148232238611\n",
      "124 50 -0.077064588666 0.000863493355647 0.0014817118092\n",
      "valid_acc 98.17666666666666\n",
      "125 0 -0.0486226566136 0.000917722575315 0.00147962550954\n",
      "125 10 -0.083194501698 0.000937767461062 0.00147887654654\n",
      "125 20 -0.0488283261657 0.000891457154825 0.00147834477984\n",
      "125 30 -0.0579899437726 0.000931356779971 0.00147704122557\n",
      "125 40 -0.0519074313343 0.000949886201767 0.00147593307787\n",
      "125 50 -0.05057810992 0.000936999836526 0.00147476719417\n",
      "valid_acc 98.16\n",
      "126 0 -0.0639023706317 0.000853568559177 0.00147218947607\n",
      "126 10 -0.0797488465905 0.000904710356956 0.00147138662255\n",
      "126 20 -0.0500069111586 0.000909812819293 0.0014709935299\n",
      "126 30 -0.0672949329019 0.000941091416159 0.00147067521786\n",
      "126 40 -0.0468855239451 0.000938603350448 0.00146944357506\n",
      "126 50 -0.0746327862144 0.000897770209232 0.0014699785626\n",
      "valid_acc 98.17\n",
      "127 0 -0.0500261858106 0.000906795548719 0.0014694734605\n",
      "127 10 -0.0837894529104 0.000947332080587 0.00146997272959\n",
      "127 20 -0.0476412586868 0.000923283368855 0.00146885756829\n",
      "127 30 -0.0526360720396 0.000958632221237 0.00147031577205\n",
      "127 40 -0.0553583055735 0.000977993502289 0.00146955507167\n",
      "127 50 -0.0520910173655 0.000983383636599 0.00146752563848\n",
      "valid_acc 98.175\n",
      "128 0 -0.0468688122928 0.00100388116162 0.00146616943893\n",
      "128 10 -0.0380031429231 0.000970854597291 0.00146462580926\n",
      "128 20 -0.0601692013443 0.000951968420507 0.00146272797975\n",
      "128 30 -0.0720954462886 0.00093655415686 0.00146202287028\n",
      "128 40 -0.0468284785748 0.000923944769686 0.00146146453759\n",
      "128 50 -0.0559154488146 0.000906014217657 0.00146165689615\n",
      "valid_acc 98.15666666666667\n",
      "129 0 -0.0593315213919 0.000913395084767 0.00146170237342\n",
      "129 10 -0.0416588671505 0.000904416646664 0.00146046726726\n",
      "129 20 -0.0553086549044 0.000930107995267 0.00146086964891\n",
      "129 30 -0.0571566745639 0.000932546066348 0.00145990974352\n",
      "129 40 -0.0691854506731 0.000925317982284 0.0014582186128\n",
      "129 50 -0.0552246049047 0.000965077841309 0.00145767255141\n",
      "valid_acc 98.20833333333333\n",
      "130 0 -0.0432448498905 0.00092993367754 0.0014567601132\n",
      "130 10 -0.0491031520069 0.000953161130167 0.00145572346841\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130 20 -0.0604798309505 0.00086817886067 0.00145388007167\n",
      "130 30 -0.0587204732001 0.000880146961633 0.00145530190804\n",
      "130 40 -0.0550262071192 0.000877560659967 0.00145611823889\n",
      "130 50 -0.0568213164806 0.000830259123293 0.00145446753167\n",
      "valid_acc 98.15333333333334\n",
      "131 0 -0.0379025191069 0.000865505400295 0.00145409521218\n",
      "131 10 -0.054735083133 0.000819740570391 0.00145452113704\n",
      "131 20 -0.0565762966871 0.000811922366819 0.00145383574788\n",
      "131 30 -0.0510967709124 0.000825840424633 0.00145234886141\n",
      "131 40 -0.0445847995579 0.000842794823152 0.00145111062573\n",
      "131 50 -0.0704174637794 0.000851216126938 0.00145039319535\n",
      "valid_acc 98.21833333333333\n",
      "132 0 -0.0531365945935 0.000811932867921 0.00144929997268\n",
      "132 10 -0.0517261847854 0.000864895013253 0.00144883069485\n",
      "132 20 -0.0414656028152 0.000834906499722 0.00144795880913\n",
      "132 30 -0.0430491343141 0.00082664917777 0.0014465132805\n",
      "132 40 -0.0662213861942 0.000841894039614 0.00144480124347\n",
      "132 50 -0.0427861809731 0.000824715165048 0.00144299235052\n",
      "valid_acc 98.2\n",
      "133 0 -0.0435525923967 0.000828925735707 0.00144177674567\n",
      "133 10 -0.0827163383365 0.000869581501765 0.00144103421105\n",
      "133 20 -0.0615793131292 0.00084856131495 0.00143856063147\n",
      "133 30 -0.0579777061939 0.000889173450718 0.00143671765872\n",
      "133 40 -0.050244640559 0.000854203557871 0.00143549613335\n",
      "133 50 -0.0476381406188 0.00084188853857 0.00143574650943\n",
      "valid_acc 98.18166666666667\n",
      "134 0 -0.0635540485382 0.000902019355222 0.00143590963733\n",
      "134 10 -0.042935859412 0.000836536823122 0.00143531893459\n",
      "134 20 -0.0439970381558 0.000875449225764 0.00143421467398\n",
      "134 30 -0.0370463840663 0.000896943980413 0.00143362921043\n",
      "134 40 -0.0681588724256 0.000833707639298 0.0014303532886\n",
      "134 50 -0.0608901679516 0.000815960033827 0.00143067617273\n",
      "valid_acc 98.225\n",
      "135 0 -0.0619049444795 0.000844187228049 0.00143035998358\n",
      "135 10 -0.0347783118486 0.000870215386563 0.00143090567156\n",
      "135 20 -0.0506092719734 0.000846158090151 0.00143075197625\n",
      "135 30 -0.0432473383844 0.000834863485171 0.00143086034358\n",
      "135 40 -0.0498476400971 0.00086003521492 0.00143070164275\n",
      "135 50 -0.0600943267345 0.000884568067986 0.00142920345835\n",
      "valid_acc 98.25\n",
      "best valid_acc 98.25\n",
      "136 0 -0.0592861659825 0.000848830296654 0.00142944088335\n",
      "136 10 -0.0629855245352 0.000871746343817 0.00142785700143\n",
      "136 20 -0.0453311614692 0.000902844538075 0.00142761046389\n",
      "136 30 -0.0401224680245 0.000845939884956 0.00142685512919\n",
      "136 40 -0.0696270316839 0.000858999923587 0.00142727544576\n",
      "136 50 -0.0461598746479 0.000820548697457 0.00142667425738\n",
      "valid_acc 98.225\n",
      "137 0 -0.0468771792948 0.000827802282619 0.00142794061327\n",
      "137 10 -0.0625063478947 0.00082026757996 0.00142816455926\n",
      "137 20 -0.0516846068203 0.000798345215958 0.00143092924488\n",
      "137 30 -0.0551550127566 0.000772385713143 0.00143083987484\n",
      "137 40 -0.0755837112665 0.000838864700971 0.0014302982986\n",
      "137 50 -0.0656704679132 0.000835050174528 0.00143007270263\n",
      "valid_acc 98.22\n",
      "138 0 -0.0477900877595 0.000823958669583 0.00142813546339\n",
      "138 10 -0.0850610285997 0.0008390113828 0.00142658205869\n",
      "138 20 -0.0516856350005 0.00081950768841 0.00142568724707\n",
      "138 30 -0.0677756592631 0.000820787670253 0.00142578596017\n",
      "138 40 -0.0521739162505 0.000822329719593 0.00142519281068\n",
      "138 50 -0.0523561351001 0.000849909564355 0.00142523889319\n",
      "valid_acc 98.22833333333332\n",
      "139 0 -0.0670092180371 0.000806155864701 0.00142347599214\n",
      "139 10 -0.0498820953071 0.000856380059083 0.00142295687321\n",
      "139 20 -0.0370899438858 0.000850806752693 0.00142122884292\n",
      "139 30 -0.0432335361838 0.000841016844231 0.00141976530958\n",
      "139 40 -0.050335124135 0.000856523477836 0.0014198875526\n",
      "139 50 -0.0524032227695 0.000855443957272 0.00141996996347\n",
      "valid_acc 98.22166666666666\n",
      "140 0 -0.0457323379815 0.000879064062711 0.0014204505704\n",
      "140 10 -0.0548699088395 0.00086326506696 0.00142053655508\n",
      "140 20 -0.0584314651787 0.000839993265545 0.00141921039014\n",
      "140 30 -0.0450072064996 0.000813552146804 0.00141964364045\n",
      "140 40 -0.0542894974351 0.000882583015955 0.00141795268398\n",
      "140 50 -0.0392369292676 0.000863211349973 0.00141761462622\n",
      "valid_acc 98.195\n",
      "141 0 -0.0632042363286 0.000816767576025 0.00141620792373\n",
      "141 10 -0.0385671928525 0.000846210549797 0.00141611053303\n",
      "141 20 -0.0566364377737 0.000871355232244 0.00141532096836\n",
      "141 30 -0.0614061057568 0.000868429062339 0.0014145333103\n",
      "141 40 -0.0446271412075 0.000848109073764 0.00141322250802\n",
      "141 50 -0.0577774122357 0.000875138238109 0.0014117346532\n",
      "valid_acc 98.25666666666667\n",
      "best valid_acc 98.25666666666667\n",
      "142 0 -0.0692539587617 0.000824007535429 0.00141030650029\n",
      "142 10 -0.0687352865934 0.000842052106924 0.0014096591242\n",
      "142 20 -0.0734281912446 0.000819485949909 0.00140832910439\n",
      "142 30 -0.085573412478 0.00088771806557 0.00140960913744\n",
      "142 40 -0.0529267564416 0.000836292518824 0.00140757731595\n",
      "142 50 -0.0460841581225 0.000839108365125 0.00140656727907\n",
      "valid_acc 98.235\n",
      "143 0 -0.0794718191028 0.000872661730089 0.00140379985286\n",
      "143 10 -0.0774687901139 0.000861326145566 0.0014022254064\n",
      "143 20 -0.0423369519413 0.000858452738659 0.00140153548548\n",
      "143 30 -0.0535514205694 0.000833127413838 0.00140070335357\n",
      "143 40 -0.0348535217345 0.000892067343616 0.00139967050614\n",
      "143 50 -0.0548965297639 0.000881019730641 0.00139728921689\n",
      "valid_acc 98.28\n",
      "best valid_acc 98.28\n",
      "144 0 -0.0531147681177 0.000876755434282 0.00139904739366\n",
      "144 10 -0.0727975517511 0.000814389153282 0.00139839598574\n",
      "144 20 -0.0325495377183 0.000837181074959 0.00139706007748\n",
      "144 30 -0.070158533752 0.000831035251174 0.00139590235339\n",
      "144 40 -0.0641935840249 0.000892859645767 0.00139604099643\n",
      "144 50 -0.0479542315006 0.000850402129422 0.00139479297641\n",
      "valid_acc 98.27666666666667\n",
      "145 0 -0.0551719516516 0.000856142978517 0.00139364702193\n",
      "145 10 -0.0478553511202 0.000840439606512 0.00139389464742\n",
      "145 20 -0.0568789541721 0.00083273893452 0.00139304641093\n",
      "145 30 -0.0771863684058 0.000839004639963 0.00139125174865\n",
      "145 40 -0.0653681308031 0.000785487155717 0.00139042519728\n",
      "145 50 -0.0506055988371 0.000821849469062 0.00139051132874\n",
      "valid_acc 98.26833333333333\n",
      "146 0 -0.0736427530646 0.000867281914047 0.00138942157473\n",
      "146 10 -0.0507362224162 0.000838371311173 0.00139035945877\n",
      "146 20 -0.0549088530242 0.000853826254938 0.00138798251611\n",
      "146 30 -0.0649789050221 0.000860351144674 0.00138575802785\n",
      "146 40 -0.0657670348883 0.000847286763336 0.00138482879024\n",
      "146 50 -0.0565243475139 0.000834146232448 0.00138487652701\n",
      "valid_acc 98.25333333333333\n",
      "147 0 -0.0281095672399 0.000862693990976 0.0013855980532\n",
      "147 10 -0.0437469370663 0.000860429688583 0.00138420068519\n",
      "147 20 -0.0515654869378 0.000896346548027 0.00138253921284\n",
      "147 30 -0.0571266375482 0.000897837465906 0.00138124340176\n",
      "147 40 -0.0439893826842 0.000881136901988 0.00138084021267\n",
      "147 50 -0.0725200921297 0.000880345542174 0.00137949753314\n",
      "valid_acc 98.23666666666668\n",
      "148 0 -0.0641972720623 0.000928272499988 0.00138003126037\n",
      "148 10 -0.0594927482307 0.000904794116332 0.00137993845394\n",
      "148 20 -0.0665240064263 0.000902261456189 0.00137874986295\n",
      "148 30 -0.0470177605748 0.000923904496712 0.00137843148915\n",
      "148 40 -0.0615574605763 0.000886185686557 0.00137679969952\n",
      "148 50 -0.0553658492863 0.000921820073382 0.00137608463809\n",
      "valid_acc 98.21833333333333\n",
      "149 0 -0.0625844597816 0.000927943966688 0.0013772601489\n",
      "149 10 -0.0467022620142 0.000956052768022 0.00137681936682\n",
      "149 20 -0.0387440361083 0.000959552292474 0.00137525509465\n",
      "149 30 -0.048654448241 0.000931684748844 0.00137476988468\n",
      "149 40 -0.0622861683369 0.000916770351304 0.00137257791929\n",
      "149 50 -0.0600502751768 0.000886746297916 0.00137010496765\n",
      "valid_acc 98.23666666666668\n",
      "150 0 -0.0416145995259 0.000880307137691 0.00136922968921\n",
      "150 10 -0.056270994246 0.00086441215287 0.0013678959798\n",
      "150 20 -0.0593546703458 0.000883316417742 0.00136725356552\n",
      "150 30 -0.0512640401721 0.00085595231601 0.00136633939732\n",
      "150 40 -0.0612505562603 0.000891243831737 0.00136451713142\n",
      "150 50 -0.0475879609585 0.000878113181719 0.00136318242325\n",
      "valid_acc 98.235\n",
      "151 0 -0.0678986683488 0.000900347620755 0.00136209641135\n",
      "151 10 -0.0689625665545 0.000873214592864 0.00135961897962\n",
      "151 20 -0.0607396773994 0.00088844028149 0.00135832783415\n",
      "151 30 -0.0511861853302 0.00093426043351 0.00135797574903\n",
      "151 40 -0.0575734190643 0.000898574763944 0.0013577122296\n",
      "151 50 -0.0402120500803 0.000886651090028 0.0013570071452\n",
      "valid_acc 98.24833333333333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152 0 -0.0514714568853 0.000876312943036 0.00135715858157\n",
      "152 10 -0.0459122695029 0.000915753740334 0.00135585178092\n",
      "152 20 -0.0612772218883 0.00089560732777 0.0013565657805\n",
      "152 30 -0.0727086514235 0.000908773148453 0.00135681761297\n",
      "152 40 -0.0557520873845 0.000904829769985 0.00135655875222\n",
      "152 50 -0.0371167771518 0.000897012961126 0.00135647267128\n",
      "valid_acc 98.25333333333333\n",
      "153 0 -0.0732221901417 0.000868551480532 0.00135555031315\n",
      "153 10 -0.030618423596 0.00091822367954 0.00135407110072\n",
      "153 20 -0.0453856512904 0.000883491310067 0.00135299255999\n",
      "153 30 -0.0422092378139 0.000946581091616 0.00135272564787\n",
      "153 40 -0.0665907040238 0.000906939815556 0.00135165521483\n",
      "153 50 -0.0795193836093 0.000863634493359 0.0013508647911\n",
      "valid_acc 98.26833333333333\n",
      "154 0 -0.0536656007171 0.000853769863079 0.00135006670751\n",
      "154 10 -0.0649991855025 0.000932211747687 0.00134901274442\n",
      "154 20 -0.067850984633 0.000871738017735 0.00134804485224\n",
      "154 30 -0.05040660128 0.000865354431528 0.00134984562278\n",
      "154 40 -0.0436394736171 0.00092723123316 0.00134952384904\n",
      "154 50 -0.0744359493256 0.000897592586469 0.00134970374975\n",
      "valid_acc 98.26166666666667\n",
      "155 0 -0.0365789495409 0.000910337176827 0.00134912370451\n",
      "155 10 -0.0527518466115 0.000907790045027 0.00134914503141\n",
      "155 20 -0.0372278876603 0.000886603125196 0.00134745260247\n",
      "155 30 -0.0500158518553 0.000868163820153 0.00134656465097\n",
      "155 40 -0.0382633209229 0.000905553953109 0.00134674148447\n",
      "155 50 -0.0470318645239 0.000922258896106 0.0013463317191\n",
      "valid_acc 98.26833333333333\n",
      "156 0 -0.0641775280237 0.00090339045036 0.00134551821942\n",
      "156 10 -0.053704995662 0.000948232686155 0.00134516252838\n",
      "156 20 -0.0443421825767 0.000934255596908 0.00134441184719\n",
      "156 30 -0.053166039288 0.000928733775567 0.00134402986879\n",
      "156 40 -0.0504281930625 0.000868137391038 0.00134296522659\n",
      "156 50 -0.0517241768539 0.00088311718092 0.00134261096966\n",
      "valid_acc 98.26166666666667\n",
      "157 0 -0.0519532077014 0.000887010550307 0.00134103903261\n",
      "157 10 -0.0535536259413 0.000908222846324 0.00133971033055\n",
      "157 20 -0.0474943034351 0.000875404315638 0.00133879164922\n",
      "157 30 -0.0398260131478 0.000865557207286 0.00133685755633\n",
      "157 40 -0.0640741884708 0.000899348107656 0.00133637832042\n",
      "157 50 -0.0393374450505 0.000848390117567 0.00133648691218\n",
      "valid_acc 98.26833333333333\n",
      "158 0 -0.0397514328361 0.000873805828496 0.00133627920861\n",
      "158 10 -0.0526900999248 0.000896431674344 0.00133560300346\n",
      "158 20 -0.0475940704346 0.00086449675319 0.00133510021049\n",
      "158 30 -0.0554804392159 0.000876088814169 0.0013341919297\n",
      "158 40 -0.0576372481883 0.000857406235863 0.00133290224833\n",
      "158 50 -0.0334681235254 0.000871129166894 0.00133124787254\n",
      "valid_acc 98.275\n",
      "159 0 -0.0589462071657 0.000887153241394 0.00132980278669\n",
      "159 10 -0.0611152611673 0.000868834629263 0.00132806472622\n",
      "159 20 -0.0461984761059 0.000830464674602 0.00132669653543\n",
      "159 30 -0.0592455528677 0.000833248258689 0.00132545907837\n",
      "159 40 -0.0544679276645 0.000854740529191 0.00132375113972\n",
      "159 50 -0.0405938923359 0.000852620309939 0.0013244279749\n",
      "valid_acc 98.26\n",
      "160 0 -0.0441596917808 0.000822745204225 0.00132366153764\n",
      "160 10 -0.0784973353148 0.000881899449709 0.00132451720025\n",
      "160 20 -0.0526842586696 0.000882834863872 0.00132328593784\n",
      "160 30 -0.0557131581008 0.000845126296274 0.00132223194149\n",
      "160 40 -0.0359999462962 0.000851800834704 0.00132174923064\n",
      "160 50 -0.0522544495761 0.000810212273329 0.00132040815475\n",
      "valid_acc 98.27166666666668\n",
      "161 0 -0.037908423692 0.000867349505996 0.00131955106923\n",
      "161 10 -0.06642575562 0.000894465388601 0.00131797899395\n",
      "161 20 -0.0649001449347 0.000880411927377 0.00131702814513\n",
      "161 30 -0.0566593967378 0.000911674641612 0.00131656246472\n",
      "161 40 -0.0564725287259 0.000871550434908 0.00131652759064\n",
      "161 50 -0.0548553168774 0.000900400879642 0.00131616169151\n",
      "valid_acc 98.26\n",
      "162 0 -0.0505974404514 0.000883012399051 0.00131536514972\n",
      "162 10 -0.0280693732202 0.000868342555668 0.00131476976124\n",
      "162 20 -0.0478326752782 0.000919059483559 0.00131406438888\n",
      "162 30 -0.0661531016231 0.00095081457021 0.00131449885982\n",
      "162 40 -0.0650525614619 0.000915297483331 0.00131469516441\n",
      "162 50 -0.0442657135427 0.000885867750001 0.00131281949088\n",
      "valid_acc 98.29666666666667\n",
      "best valid_acc 98.29666666666667\n",
      "163 0 -0.0747216716409 0.000879260482977 0.00131269045216\n",
      "163 10 -0.0462876670063 0.000886153154117 0.00131167354613\n",
      "163 20 -0.0386445336044 0.000882182769189 0.00131149782465\n",
      "163 30 -0.0545808896422 0.000920333236604 0.001310959236\n",
      "163 40 -0.0525530315936 0.00090384830013 0.00131018122087\n",
      "163 50 -0.065926246345 0.000900463346092 0.00131039947162\n",
      "valid_acc 98.285\n",
      "164 0 -0.0540560185909 0.000946468459179 0.00130966301733\n",
      "164 10 -0.0511809960008 0.000885548172453 0.00130996207392\n",
      "164 20 -0.0501943677664 0.000897526006366 0.00130787480518\n",
      "164 30 -0.0583224631846 0.000906009500549 0.00130721206031\n",
      "164 40 -0.0615201070905 0.000891949157196 0.00130805355047\n",
      "164 50 -0.0630939602852 0.000910225577826 0.00130639468\n",
      "valid_acc 98.25666666666667\n",
      "165 0 -0.0504381135106 0.000909456258904 0.0013053660258\n",
      "165 10 -0.0530672706664 0.000904180021174 0.00130383103578\n",
      "165 20 -0.0651647523046 0.000920019804513 0.00130196794263\n",
      "165 30 -0.0451746061444 0.000840576705374 0.00130041631143\n",
      "165 40 -0.0391593426466 0.000900545580191 0.0012995986795\n",
      "165 50 -0.0628209412098 0.000912902293848 0.00129920662005\n",
      "valid_acc 98.31333333333333\n",
      "best valid_acc 98.31333333333333\n",
      "166 0 -0.0577647536993 0.000867701390987 0.00129853808693\n",
      "166 10 -0.0652845054865 0.000915932689281 0.00129869236841\n",
      "166 20 -0.0507251359522 0.000865666442996 0.00129898661773\n",
      "166 30 -0.041658077389 0.000887870127949 0.00129795655464\n",
      "166 40 -0.0542897433043 0.000879395183496 0.00129825174768\n",
      "166 50 -0.0594994351268 0.000871055862398 0.00129679403838\n",
      "valid_acc 98.29833333333333\n",
      "167 0 -0.0537619553506 0.000924807787477 0.00129492394927\n",
      "167 10 -0.0463777668774 0.000928754436705 0.00129443069898\n",
      "167 20 -0.0541921332479 0.000900418984858 0.0012926955385\n",
      "167 30 -0.0501507334411 0.00089165616987 0.00129322692043\n",
      "167 40 -0.0375060774386 0.000908186580145 0.00129239372592\n",
      "167 50 -0.0568967908621 0.000911996769728 0.0012918812641\n",
      "valid_acc 98.29\n",
      "168 0 -0.0591671653092 0.000902331913618 0.00129114941653\n",
      "168 10 -0.0533181093633 0.000905578724766 0.00129124360928\n",
      "168 20 -0.0416429303586 0.000936072507636 0.00129055540163\n",
      "168 30 -0.0373829826713 0.00090891598614 0.00128944106295\n",
      "168 40 -0.0517991632223 0.000889954417289 0.00128769890812\n",
      "168 50 -0.0323797278106 0.000889961543375 0.0012867857451\n",
      "valid_acc 98.28\n",
      "169 0 -0.0349053964019 0.000910341455035 0.00128758375306\n",
      "169 10 -0.0471982061863 0.000897183191061 0.00128571427912\n",
      "169 20 -0.0604808963835 0.000896866898732 0.00128586683231\n",
      "169 30 -0.0446158833802 0.000859833446329 0.00128416833473\n",
      "169 40 -0.0463829226792 0.000872932656478 0.00128361062895\n",
      "169 50 -0.0499342307448 0.000883723062109 0.00128296380646\n",
      "valid_acc 98.265\n",
      "170 0 -0.0636169016361 0.000893941412531 0.00128330709184\n",
      "170 10 -0.0479315333068 0.000888884490188 0.00128277072677\n",
      "170 20 -0.0394618064165 0.00086527797441 0.00128071477705\n",
      "170 30 -0.0309218876064 0.000883679481285 0.00128073015659\n",
      "170 40 -0.0685916915536 0.000891690559053 0.00127964795295\n",
      "170 50 -0.0658219456673 0.000881333893071 0.00127778291556\n",
      "valid_acc 98.295\n",
      "171 0 -0.031484592706 0.00087695115127 0.0012777559831\n",
      "171 10 -0.0569985471666 0.000861520778062 0.00127857923711\n",
      "171 20 -0.043642539531 0.000879488707673 0.00127750055964\n",
      "171 30 -0.0482472516596 0.000792150373691 0.00127696293559\n",
      "171 40 -0.0567648261786 0.000837342822301 0.00127690199509\n",
      "171 50 -0.0427005626261 0.000842949374484 0.00127673132082\n",
      "valid_acc 98.28666666666666\n",
      "172 0 -0.0487403199077 0.000851407103215 0.00127678322204\n",
      "172 10 -0.0539409480989 0.000851923648514 0.00127607002542\n",
      "172 20 -0.0494241937995 0.000816059387607 0.00127710555463\n",
      "172 30 -0.0430926606059 0.000830076013202 0.0012757505976\n",
      "172 40 -0.0506930761039 0.000874381398061 0.00127409383408\n",
      "172 50 -0.0538383573294 0.000835358027071 0.00127407448471\n",
      "valid_acc 98.235\n",
      "173 0 -0.0308667831123 0.000845975798649 0.00127204772959\n",
      "173 10 -0.050987072289 0.00088187029927 0.00127071560335\n",
      "173 20 -0.0621209293604 0.000849895824195 0.00127079784401\n",
      "173 30 -0.0708320885897 0.000817047829268 0.00127045035087\n",
      "173 40 -0.0783835202456 0.00083792769537 0.00126942474109\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "173 50 -0.0474831648171 0.00080795734194 0.00126842552536\n",
      "valid_acc 98.28833333333333\n",
      "174 0 -0.0477095209062 0.00083697961909 0.00126784047137\n",
      "174 10 -0.0536008737981 0.000836621086922 0.00126669675002\n",
      "174 20 -0.053429376334 0.000805206495312 0.00126616859063\n",
      "174 30 -0.0383417271078 0.000834426606725 0.00126455902387\n",
      "174 40 -0.056108020246 0.000821800097136 0.00126467802958\n",
      "174 50 -0.0518060848117 0.000790490813534 0.00126437058862\n",
      "valid_acc 98.31\n",
      "175 0 -0.0593904405832 0.000799294471051 0.00126401246379\n",
      "175 10 -0.045736502856 0.0008018735391 0.00126339634444\n",
      "175 20 -0.0324431620538 0.000846627189731 0.00126223262439\n",
      "175 30 -0.0604943707585 0.000826791312576 0.00126125241308\n",
      "175 40 -0.0627987980843 0.000804175525206 0.00126083844047\n",
      "175 50 -0.0545062199235 0.000787666965506 0.00126011890642\n",
      "valid_acc 98.31\n",
      "176 0 -0.0411993414164 0.000797386619866 0.00125907342602\n",
      "176 10 -0.0470187626779 0.000809215320605 0.00125841808311\n",
      "176 20 -0.0440991520882 0.000763487277486 0.00125854431006\n",
      "176 30 -0.0433922335505 0.000795839261837 0.00125606548858\n",
      "176 40 -0.0549840666354 0.000791263063247 0.00125391471312\n",
      "176 50 -0.0467537045479 0.000766123018675 0.00125327597107\n",
      "valid_acc 98.30166666666666\n",
      "177 0 -0.0539192669094 0.000748242950114 0.0012535012535\n",
      "177 10 -0.0339245423675 0.000764838982086 0.00125290359073\n",
      "177 20 -0.0433500520885 0.000742849204341 0.00125153311066\n",
      "177 30 -0.0601906441152 0.000750666912681 0.00125171597145\n",
      "177 40 -0.0552629828453 0.000773667942924 0.00125080823595\n",
      "177 50 -0.0412181280553 0.000749350337017 0.00125022653064\n",
      "valid_acc 98.30333333333333\n",
      "178 0 -0.0426410101354 0.000790674277884 0.00124958117796\n",
      "178 10 -0.0335296690464 0.000726389395708 0.001249103664\n",
      "178 20 -0.046478189528 0.000733613746928 0.00124879307283\n",
      "178 30 -0.0404225997627 0.00076674857297 0.00124663014222\n",
      "178 40 -0.0403753519058 0.000755426928675 0.0012456522883\n",
      "178 50 -0.0413160063326 0.000749211062828 0.00124501087661\n",
      "valid_acc 98.33666666666666\n",
      "best valid_acc 98.33666666666666\n",
      "179 0 -0.0578407496214 0.000761592377057 0.00124461139041\n",
      "179 10 -0.0582279525697 0.000748537635004 0.0012439152964\n",
      "179 20 -0.0620231740177 0.000766792133985 0.00124271053555\n",
      "179 30 -0.0670010820031 0.000771679991448 0.00124189246241\n",
      "179 40 -0.0513836406171 0.000778698890603 0.00124123275488\n",
      "179 50 -0.0589698031545 0.000746217502735 0.00124055294759\n",
      "valid_acc 98.31666666666666\n",
      "180 0 -0.0496127679944 0.000770090538822 0.00123957117986\n",
      "180 10 -0.0473784059286 0.000740213855823 0.0012389829117\n",
      "180 20 -0.0363952592015 0.000758513413111 0.00123913471099\n",
      "180 30 -0.0535427480936 0.000793446683927 0.0012387876089\n",
      "180 40 -0.0613001734018 0.000782981522764 0.00123882878871\n",
      "180 50 -0.0574913024902 0.000769046218073 0.00123825112401\n",
      "valid_acc 98.35666666666667\n",
      "best valid_acc 98.35666666666667\n",
      "181 0 -0.0463977009058 0.000786768212012 0.00123805737875\n",
      "181 10 -0.0451248213649 0.000798606569221 0.0012379665715\n",
      "181 20 -0.042706143111 0.000749390737626 0.00123668502579\n",
      "181 30 -0.043582059443 0.000783265886031 0.00123565264321\n",
      "181 40 -0.0613915100694 0.000798675500738 0.00123496694817\n",
      "181 50 -0.058913551271 0.000763463029781 0.00123494382985\n",
      "valid_acc 98.345\n",
      "182 0 -0.0740839838982 0.000784699808986 0.00123534328318\n",
      "182 10 -0.0519331134856 0.000736718479247 0.00123493136859\n",
      "182 20 -0.0699543356895 0.00075523476884 0.00123409350154\n",
      "182 30 -0.040231578052 0.000723356194869 0.00123269568957\n",
      "182 40 -0.0373398959637 0.000774818874261 0.00123271110177\n",
      "182 50 -0.0489723496139 0.000748692761742 0.00123213026033\n",
      "valid_acc 98.335\n",
      "183 0 -0.0708918124437 0.00072705210978 0.00123270176078\n",
      "183 10 -0.0459363982081 0.000746429957971 0.00123208986203\n",
      "183 20 -0.0477069318295 0.000722163111882 0.00123055450453\n",
      "183 30 -0.0572790019214 0.000690848009514 0.00122980746281\n",
      "183 40 -0.0329936742783 0.000735851007544 0.00122787203883\n",
      "183 50 -0.0672030597925 0.00070715076085 0.00122785614512\n",
      "valid_acc 98.31833333333333\n",
      "184 0 -0.044312722981 0.000739536311706 0.00122718793914\n",
      "184 10 -0.0811607912183 0.000778690225299 0.00122655389786\n",
      "184 20 -0.0747340992093 0.000761172398799 0.00122585271347\n",
      "184 30 -0.0431657321751 0.000708983343942 0.00122448722317\n",
      "184 40 -0.0389268510044 0.0007134473778 0.00122453549663\n",
      "184 50 -0.0381723493338 0.000723246834334 0.00122304895333\n",
      "valid_acc 98.34333333333333\n",
      "185 0 -0.0574692003429 0.000760293778809 0.00122292812228\n",
      "185 10 -0.0584839396179 0.000761790030468 0.00122224353108\n",
      "185 20 -0.0479243434966 0.000767104527897 0.00122187999868\n",
      "185 30 -0.0586817413568 0.000743933851219 0.00122202424011\n",
      "185 40 -0.052028875798 0.000699512054654 0.00122221241876\n",
      "185 50 -0.0832655206323 0.000711596803225 0.00122220339585\n",
      "valid_acc 98.315\n",
      "186 0 -0.0500752255321 0.000727990096874 0.00122115418609\n",
      "186 10 -0.0406265631318 0.000738485184541 0.00122031271342\n",
      "186 20 -0.0668438002467 0.000715922583266 0.0012193775461\n",
      "186 30 -0.058911472559 0.000727236260003 0.00121754958586\n",
      "186 40 -0.041939586401 0.00075713775805 0.00121686783162\n",
      "186 50 -0.0548929199576 0.000723994065687 0.00121751638311\n",
      "valid_acc 98.35166666666667\n",
      "187 0 -0.0720793381333 0.000779791563928 0.00121675555162\n",
      "187 10 -0.0487797707319 0.000748791576704 0.00121576895118\n",
      "187 20 -0.0406708829105 0.000751935153389 0.00121552698332\n",
      "187 30 -0.0526085831225 0.000773033591389 0.00121519904348\n",
      "187 40 -0.0452137254179 0.000796935702001 0.0012156824772\n",
      "187 50 -0.0654550492764 0.00073973616003 0.00121520342313\n",
      "valid_acc 98.33\n",
      "188 0 -0.0582276545465 0.000761173886869 0.00121285144603\n",
      "188 10 -0.0556368157268 0.000745903983354 0.00121288022479\n",
      "188 20 -0.0588191412389 0.000755748747222 0.00121286655752\n",
      "188 30 -0.0569982230663 0.000738458940251 0.00121244598938\n",
      "188 40 -0.0743990838528 0.000761987029647 0.00121149980951\n",
      "188 50 -0.065574683249 0.00077762337152 0.00121089602175\n",
      "valid_acc 98.32\n",
      "189 0 -0.0437116138637 0.00078888140893 0.00120973989964\n",
      "189 10 -0.0567984059453 0.000742566332919 0.00120933990181\n",
      "189 20 -0.0469182096422 0.000778409735886 0.00120938595415\n",
      "189 30 -0.048832770437 0.000789817315442 0.00120947805152\n",
      "189 40 -0.0427378043532 0.000767667973274 0.00120918809151\n",
      "189 50 -0.0581948123872 0.000772867735898 0.00120905143317\n",
      "valid_acc 98.33166666666666\n",
      "190 0 -0.0383418500423 0.000793265800657 0.00120896851233\n",
      "190 10 -0.0454430505633 0.000763204079821 0.0012084448247\n",
      "190 20 -0.0624247938395 0.00079942503312 0.00120916601642\n",
      "190 30 -0.0442630723119 0.000812917055005 0.00120844311422\n",
      "190 40 -0.053815510124 0.000788731942139 0.00120798623662\n",
      "190 50 -0.0360199660063 0.000777523205184 0.00120759671956\n",
      "valid_acc 98.33\n",
      "191 0 -0.0496319122612 0.000740282238082 0.00120777441159\n",
      "191 10 -0.0573468916118 0.000757482227782 0.00120723888259\n",
      "191 20 -0.0495000593364 0.000779897835829 0.00120708976275\n",
      "191 30 -0.0516080223024 0.000791567670107 0.00120675380849\n",
      "191 40 -0.0557386428118 0.000741442248809 0.00120640257765\n",
      "191 50 -0.0511899106205 0.000809706476314 0.0012051259229\n",
      "valid_acc 98.32333333333332\n",
      "192 0 -0.0283121727407 0.000759378756333 0.00120388411684\n",
      "192 10 -0.0581455677748 0.000782818178423 0.00120324874927\n",
      "192 20 -0.045495506376 0.000738374144138 0.00120177290832\n",
      "192 30 -0.0436939522624 0.000765754452316 0.00120163490544\n",
      "192 40 -0.0431192927063 0.000761414881196 0.00120027180094\n",
      "192 50 -0.0674074813724 0.00077669273952 0.00119984295545\n",
      "valid_acc 98.345\n",
      "193 0 -0.0526796653867 0.000800167663726 0.0011982708987\n",
      "193 10 -0.0411231964827 0.000796059404549 0.00119829470597\n",
      "193 20 -0.0506014712155 0.000777297813753 0.00119771526633\n",
      "193 30 -0.0628296211362 0.000790418153375 0.00119711385232\n",
      "193 40 -0.0550332702696 0.000802120933525 0.00119669723187\n",
      "193 50 -0.0570587478578 0.000751824378528 0.00119528125654\n",
      "valid_acc 98.32666666666667\n",
      "194 0 -0.0552328340709 0.000776879316239 0.00119476144852\n",
      "194 10 -0.0442096740007 0.000753933542581 0.00119390077769\n",
      "194 20 -0.0461954735219 0.000774908667451 0.00119362764369\n",
      "194 30 -0.0468613319099 0.000774508571173 0.00119321691757\n",
      "194 40 -0.0592838712037 0.000793845442876 0.00119315352961\n",
      "194 50 -0.0518176443875 0.000774420581761 0.00119362785456\n",
      "valid_acc 98.35166666666667\n",
      "195 0 -0.048651073128 0.000782783483412 0.00119254726322\n",
      "195 10 -0.0380353406072 0.000784610685455 0.00119199447323\n",
      "195 20 -0.055081486702 0.000788320284039 0.00119237526011\n",
      "195 30 -0.0866835042834 0.000822308614721 0.00119079615479\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195 40 -0.0401849076152 0.000797278053776 0.00118940614611\n",
      "195 50 -0.05584590137 0.000819027831252 0.00118800309985\n",
      "valid_acc 98.34333333333333\n",
      "196 0 -0.0350663922727 0.000821504748481 0.00118651557139\n",
      "196 10 -0.0254001691937 0.000809798022409 0.00118535897098\n",
      "196 20 -0.0431939214468 0.000798445720637 0.00118538135469\n",
      "196 30 -0.029971152544 0.000838492523778 0.00118487227126\n",
      "196 40 -0.0761822834611 0.000833086305961 0.00118343861587\n",
      "196 50 -0.0603621974587 0.000831327996104 0.00118182315417\n",
      "valid_acc 98.33833333333334\n",
      "197 0 -0.0636989548802 0.000788704192658 0.00118208468791\n",
      "197 10 -0.0562959723175 0.000784268143954 0.00118073950895\n",
      "197 20 -0.0352161414921 0.000825106738827 0.00117914350091\n",
      "197 30 -0.0519058182836 0.000814498752176 0.00117935072264\n",
      "197 40 -0.0449934490025 0.000802728452145 0.00117846450591\n",
      "197 50 -0.063168309629 0.000794961789941 0.00117906902568\n",
      "valid_acc 98.32666666666667\n",
      "198 0 -0.068925127387 0.000808105576929 0.00117792361095\n",
      "198 10 -0.0571372099221 0.000807256850485 0.00117668179043\n",
      "198 20 -0.0614946447313 0.000783401918491 0.00117702852396\n",
      "198 30 -0.0428161434829 0.000822644082215 0.00117606445518\n",
      "198 40 -0.0467981621623 0.000782322899297 0.00117599522015\n",
      "198 50 -0.0479153729975 0.000746796939703 0.00117585319776\n",
      "valid_acc 98.34\n",
      "199 0 -0.0521249175072 0.000784324109614 0.00117534302099\n",
      "199 10 -0.0471701622009 0.000775476115642 0.00117584615994\n",
      "199 20 -0.0420067720115 0.00081409471214 0.00117528136877\n",
      "199 30 -0.0533111654222 0.000763709664978 0.00117488191645\n",
      "199 40 -0.0439882874489 0.00077570937798 0.00117620255044\n",
      "199 50 -0.0455386191607 0.000759512579381 0.00117512599682\n",
      "valid_acc 98.34833333333334\n",
      "200 0 -0.0437854900956 0.000784017789794 0.00117520743798\n",
      "200 10 -0.0388743914664 0.000823873786535 0.00117422877014\n",
      "200 20 -0.0500965751708 0.000779603357464 0.00117436629002\n",
      "200 30 -0.055701635778 0.000808503155932 0.00117423857872\n",
      "200 40 -0.0418528914452 0.000800653407314 0.00117432954001\n",
      "200 50 -0.0625305101275 0.000810121155987 0.00117235051531\n",
      "valid_acc 98.34166666666667\n",
      "201 0 -0.0423362851143 0.000814069406257 0.00117257335066\n",
      "201 10 -0.0443955734372 0.000783577522822 0.00117163332761\n",
      "201 20 -0.046089373529 0.000805689167558 0.00117139704587\n",
      "201 30 -0.0346711017191 0.000818165594076 0.00117142301998\n",
      "201 40 -0.0791814252734 0.000839128557103 0.00116954690187\n",
      "201 50 -0.0531535334885 0.000841788397861 0.00117031151894\n",
      "valid_acc 98.345\n",
      "202 0 -0.0392623431981 0.000827604590503 0.00117035997425\n",
      "202 10 -0.0437484495342 0.000821741055265 0.00117047737492\n",
      "202 20 -0.0709912404418 0.000808460111558 0.00117004424735\n",
      "202 30 -0.063643693924 0.000833701161709 0.00116955288527\n",
      "202 40 -0.0291819907725 0.000853805869042 0.00116870324336\n",
      "202 50 -0.0508944652975 0.000762115017 0.00116877523498\n",
      "valid_acc 98.37833333333333\n",
      "best valid_acc 98.37833333333333\n",
      "203 0 -0.0450005717576 0.000834129794427 0.00116802989308\n",
      "203 10 -0.0702454671264 0.000809568363633 0.00116835986569\n",
      "203 20 -0.0435115173459 0.000816826992787 0.00116768085148\n",
      "203 30 -0.0500019788742 0.000811377579192 0.00116712551507\n",
      "203 40 -0.0606505833566 0.000784759553893 0.00116674205217\n",
      "203 50 -0.0723207294941 0.000831704653888 0.00116646894618\n",
      "valid_acc 98.36\n",
      "204 0 -0.0588092058897 0.000792887857638 0.00116551788913\n",
      "204 10 -0.0415068380535 0.000811174848798 0.00116485480282\n",
      "204 20 -0.0399731099606 0.000774327550642 0.00116500282783\n",
      "204 30 -0.0646380931139 0.000765985445968 0.00116471750853\n",
      "204 40 -0.0435872711241 0.000808489567006 0.00116378108831\n",
      "204 50 -0.0521158613265 0.00079775680208 0.0011632326615\n",
      "valid_acc 98.35000000000001\n",
      "205 0 -0.0660346522927 0.00076640228759 0.00116203451041\n",
      "205 10 -0.0334698483348 0.000789513825975 0.00116049638661\n",
      "205 20 -0.0651333034039 0.000789875869934 0.00115961984523\n",
      "205 30 -0.0386825613678 0.000781237914766 0.00116023980155\n",
      "205 40 -0.0414762869477 0.0007579554844 0.00115928583443\n",
      "205 50 -0.0431681349874 0.000793715354233 0.00116076068879\n",
      "valid_acc 98.35833333333333\n",
      "206 0 -0.042781945318 0.000791655096808 0.00116140667725\n",
      "206 10 -0.0815331190825 0.00079830429677 0.00116188686\n",
      "206 20 -0.0603085421026 0.000828866518975 0.00116198418757\n",
      "206 30 -0.0388705059886 0.000793179670502 0.00116107853578\n",
      "206 40 -0.0539342425764 0.00080115059367 0.00116132244912\n",
      "206 50 -0.0436648055911 0.000797518537968 0.00116127689127\n",
      "valid_acc 98.35000000000001\n",
      "207 0 -0.0413753502071 0.000818770319495 0.0011617312474\n",
      "207 10 -0.0684855878353 0.000751577705815 0.00116036460371\n",
      "207 20 -0.0524806827307 0.000811047432987 0.00115941767594\n",
      "207 30 -0.0546194203198 0.000771189756974 0.00116026934457\n",
      "207 40 -0.0449470840394 0.00074874087458 0.00116032628292\n",
      "207 50 -0.0481476299465 0.000810915405374 0.00115944983722\n",
      "valid_acc 98.36666666666667\n",
      "208 0 -0.047984752804 0.000797143471836 0.00115883222872\n",
      "208 10 -0.0360462777317 0.000776941039751 0.00115909079604\n",
      "208 20 -0.0469612330198 0.00081145653059 0.00115755962819\n",
      "208 30 -0.0489339865744 0.000793395325484 0.00115741258132\n",
      "208 40 -0.0485390275717 0.000810779675716 0.00115713562789\n",
      "208 50 -0.0571575313807 0.000838229804973 0.00115612085853\n",
      "valid_acc 98.35333333333334\n",
      "209 0 -0.0411420501769 0.000820601731593 0.00115524547686\n",
      "209 10 -0.0730455890298 0.000769810133884 0.00115530560692\n",
      "209 20 -0.0546400882304 0.000828262881541 0.00115567239461\n",
      "209 30 -0.0524072982371 0.000806263406078 0.00115469612895\n",
      "209 40 -0.0463518127799 0.000790916397899 0.00115409662361\n",
      "209 50 -0.0539849698544 0.00078086830118 0.00115405542409\n",
      "valid_acc 98.38\n",
      "best valid_acc 98.38\n",
      "210 0 -0.0416102185845 0.00076973276316 0.00115423797463\n",
      "210 10 -0.0518552847207 0.000761208154059 0.00115371966446\n",
      "210 20 -0.0744915381074 0.000739783914421 0.00115388384649\n",
      "210 30 -0.047107629478 0.000813370795701 0.00115337111854\n",
      "210 40 -0.0401827469468 0.000829950750436 0.00115242591224\n",
      "210 50 -0.0419842265546 0.000791687266905 0.00115232404835\n",
      "valid_acc 98.37\n",
      "211 0 -0.0523530393839 0.00080022152817 0.0011524570128\n",
      "211 10 -0.0780738145113 0.000786408635968 0.00115299736374\n",
      "211 20 -0.0354016274214 0.000793536469953 0.00115168003344\n",
      "211 30 -0.061315998435 0.000787618809138 0.00115162157122\n",
      "211 40 -0.0420774295926 0.000798796652089 0.00114988764244\n",
      "211 50 -0.0429077595472 0.000772424363026 0.00114964187571\n",
      "valid_acc 98.37166666666667\n",
      "212 0 -0.0454244539142 0.000796814621661 0.00114901013984\n",
      "212 10 -0.0459855496883 0.000790409651757 0.00114804239666\n",
      "212 20 -0.0633552372456 0.000784101188865 0.00114695068478\n",
      "212 30 -0.0573824830353 0.0008095911074 0.00114599247277\n",
      "212 40 -0.0448735691607 0.000790783840166 0.00114615288001\n",
      "212 50 -0.0403268821537 0.000784063997109 0.00114655978257\n",
      "valid_acc 98.36833333333334\n",
      "213 0 -0.0460169278085 0.000769508908255 0.0011456313323\n",
      "213 10 -0.0446665249765 0.000807314854806 0.00114487215505\n",
      "213 20 -0.0387624725699 0.000805689680081 0.00114567986541\n",
      "213 30 -0.0626838281751 0.000851499160085 0.00114365400407\n",
      "213 40 -0.0460946857929 0.000790999619932 0.0011427046601\n",
      "213 50 -0.0593145154417 0.000795104430588 0.0011419557312\n",
      "valid_acc 98.38333333333334\n",
      "best valid_acc 98.38333333333334\n",
      "214 0 -0.0441919006407 0.00078657503844 0.00114194361045\n",
      "214 10 -0.049303907901 0.000783805401093 0.00114186656742\n",
      "214 20 -0.047202296555 0.000815244334139 0.00114158568444\n",
      "214 30 -0.0278968177736 0.000802615631984 0.00114118375238\n",
      "214 40 -0.0550539530814 0.000772442219024 0.00114068392017\n",
      "214 50 -0.0605138167739 0.000765021368836 0.00114004071951\n",
      "valid_acc 98.36\n",
      "215 0 -0.0563860796392 0.000761442412402 0.00114011007117\n",
      "215 10 -0.0342409797013 0.000776606277684 0.00113861580795\n",
      "215 20 -0.0459137707949 0.000769243245083 0.00113837010104\n",
      "215 30 -0.0668857097626 0.000768017282998 0.00113729630614\n",
      "215 40 -0.038609713316 0.000773880705152 0.00113663148917\n",
      "215 50 -0.0778477564454 0.000739849453309 0.00113626315363\n",
      "valid_acc 98.35333333333334\n",
      "216 0 -0.0479951240122 0.00078202932301 0.00113487001423\n",
      "216 10 -0.0570288449526 0.00074992567017 0.00113471010314\n",
      "216 20 -0.0884046927094 0.000751918915931 0.00113358222695\n",
      "216 30 -0.0457082055509 0.000722091029467 0.00113207194446\n",
      "216 40 -0.0656186938286 0.000766213213322 0.00113126998125\n",
      "216 50 -0.0430636592209 0.000768551402608 0.00113118854549\n",
      "valid_acc 98.39666666666666\n",
      "best valid_acc 98.39666666666666\n",
      "217 0 -0.0435220561922 0.000758143816386 0.00113099626712\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217 10 -0.0331619977951 0.000745991584884 0.00113051080544\n",
      "217 20 -0.035424567759 0.000753384104668 0.00113144751067\n",
      "217 30 -0.0367377921939 0.00072147062138 0.00113148883086\n",
      "217 40 -0.0572004169226 0.00073390004988 0.00113123771878\n",
      "217 50 -0.0546262077987 0.000767081034778 0.00113069236172\n",
      "valid_acc 98.38333333333334\n",
      "218 0 -0.0520004332066 0.000746730055623 0.00113089491723\n",
      "218 10 -0.0301133580506 0.000735633103216 0.00113105254447\n",
      "218 20 -0.05264486745 0.000761892501132 0.00113064909396\n",
      "218 30 -0.0542731359601 0.00072701972617 0.00112978377118\n",
      "218 40 -0.0356832072139 0.000766315152333 0.00112935164282\n",
      "218 50 -0.0446718782187 0.00076455076966 0.00112989586504\n",
      "valid_acc 98.405\n",
      "best valid_acc 98.405\n",
      "219 0 -0.0325374901295 0.000738393857435 0.00112909371404\n",
      "219 10 -0.0382205843925 0.000751709017076 0.00112788758185\n",
      "219 20 -0.0401618108153 0.000711796707398 0.00112631300182\n",
      "219 30 -0.0359147936106 0.000765841167231 0.00112649059236\n",
      "219 40 -0.0439355559647 0.000741209477635 0.00112665628844\n",
      "219 50 -0.0567854419351 0.000734386713722 0.00112527268364\n",
      "valid_acc 98.38833333333334\n",
      "220 0 -0.0552187971771 0.000731648458631 0.00112492643228\n",
      "220 10 -0.0321407578886 0.000747947713903 0.00112384883014\n",
      "220 20 -0.0441067330539 0.000743277095197 0.00112360633405\n",
      "220 30 -0.0646108686924 0.000743514757586 0.00112131699461\n",
      "220 40 -0.0576377809048 0.000727739179617 0.00112054277475\n",
      "220 50 -0.0733422264457 0.000692103337809 0.00111969011658\n",
      "valid_acc 98.37666666666667\n",
      "221 0 -0.0492663122714 0.000743313777629 0.00111986833513\n",
      "221 10 -0.0467997342348 0.000718444490649 0.00111980494812\n",
      "221 20 -0.0509990379214 0.000732028757663 0.00111916950887\n",
      "221 30 -0.0360955148935 0.000740595031017 0.00111836874082\n",
      "221 40 -0.0427762381732 0.000741689671312 0.00111730107133\n",
      "221 50 -0.0653721913695 0.000724668389588 0.00111667497369\n",
      "valid_acc 98.39\n",
      "222 0 -0.0563725568354 0.000743966567476 0.00111610228024\n",
      "222 10 -0.0404825471342 0.00077739314895 0.00111530148186\n",
      "222 20 -0.0411101765931 0.000784955364489 0.00111369021008\n",
      "222 30 -0.0425105541945 0.000728930657615 0.00111409458966\n",
      "222 40 -0.0599929951131 0.000732123339391 0.0011137391622\n",
      "222 50 -0.0515603795648 0.00074000311347 0.00111396599654\n",
      "valid_acc 98.39\n",
      "223 0 -0.0469146743417 0.000713119089573 0.00111279406626\n",
      "223 10 -0.0510392785072 0.000724980970349 0.00111292513202\n",
      "223 20 -0.0512742102146 0.000687700293774 0.00111212765156\n",
      "223 30 -0.0503575913608 0.00073334532733 0.00111053577811\n",
      "223 40 -0.0598120465875 0.000717116863092 0.00111101677209\n",
      "223 50 -0.0422903299332 0.000744163180129 0.00111117738913\n",
      "valid_acc 98.37333333333333\n",
      "224 0 -0.0398850925267 0.000755242966933 0.00111146483672\n",
      "224 10 -0.0488148704171 0.000740888025248 0.00111116045211\n",
      "224 20 -0.0504544861615 0.000711547790436 0.00111058961504\n",
      "224 30 -0.0429727956653 0.000710710452617 0.00110926153003\n",
      "224 40 -0.0731613785028 0.000760283787685 0.0011088959278\n",
      "224 50 -0.061259418726 0.000740505619838 0.00110916283606\n",
      "valid_acc 98.42166666666667\n",
      "best valid_acc 98.42166666666667\n",
      "225 0 -0.0774942114949 0.000756224432264 0.00110820672814\n",
      "225 10 -0.0791731700301 0.000734163171909 0.0011076397843\n",
      "225 20 -0.031609620899 0.000738931494359 0.00110725084836\n",
      "225 30 -0.0390453711152 0.000700404518441 0.00110718895989\n",
      "225 40 -0.0312963239849 0.00072675951639 0.0011076688074\n",
      "225 50 -0.0526623502374 0.000702915058836 0.00110796305266\n",
      "valid_acc 98.41333333333333\n",
      "226 0 -0.0371836163104 0.000700596600074 0.00110730017536\n",
      "226 10 -0.0382770225406 0.000728500125562 0.00110581073797\n",
      "226 20 -0.0527361854911 0.000729682847103 0.00110550672879\n",
      "226 30 -0.0501923076808 0.000734871811713 0.00110472349433\n",
      "226 40 -0.0614292770624 0.000711447753696 0.00110412143521\n",
      "226 50 -0.054987154901 0.000718385059675 0.00110353758583\n",
      "valid_acc 98.43499999999999\n",
      "best valid_acc 98.43499999999999\n",
      "227 0 -0.0503088831902 0.000788567139585 0.00110343618997\n",
      "227 10 -0.046155449003 0.000731731023445 0.00110239945652\n",
      "227 20 -0.0382932759821 0.000725537474937 0.00110236467731\n",
      "227 30 -0.0544111393392 0.000768589624618 0.00110273389027\n",
      "227 40 -0.0336452722549 0.00077212152642 0.0011023558843\n",
      "227 50 -0.0402236878872 0.000754474503196 0.00110209419053\n",
      "valid_acc 98.405\n",
      "228 0 -0.0621563605964 0.000742352441464 0.00110181341433\n",
      "228 10 -0.0591439083219 0.000745114801953 0.00110068496684\n",
      "228 20 -0.0476681664586 0.000794919021298 0.00109962923257\n",
      "228 30 -0.0416237637401 0.000789282926371 0.00110009800655\n",
      "228 40 -0.0598993450403 0.000747683001161 0.00110026140034\n",
      "228 50 -0.0327912755311 0.000792634680614 0.00109968497291\n",
      "valid_acc 98.41499999999999\n",
      "229 0 -0.068613640964 0.000734699417189 0.00109855061205\n",
      "229 10 -0.042048510164 0.000729529189296 0.00109824300407\n",
      "229 20 -0.064685113728 0.000759170296633 0.00109706333871\n",
      "229 30 -0.0605503171682 0.000771950827018 0.00109579286114\n",
      "229 40 -0.0433752834797 0.000738030686868 0.00109421540166\n",
      "229 50 -0.0521956533194 0.000746436851641 0.00109394419837\n",
      "valid_acc 98.42166666666667\n",
      "230 0 -0.0524156466126 0.000722532839382 0.00109411024017\n",
      "230 10 -0.0549267120659 0.000759859437563 0.00109328533145\n",
      "230 20 -0.0262710005045 0.000730141620181 0.00109198610462\n",
      "230 30 -0.0473004542291 0.000778429008184 0.00109215473307\n",
      "230 40 -0.0436242297292 0.000750649460041 0.00109156453637\n",
      "230 50 -0.0518412701786 0.000771279115978 0.00109201442071\n",
      "valid_acc 98.405\n",
      "231 0 -0.0497645922005 0.000748337727405 0.0010917807667\n",
      "231 10 -0.0730097740889 0.000751926472218 0.00109040619992\n",
      "231 20 -0.045311216265 0.000746210389763 0.00108999804182\n",
      "231 30 -0.0617454834282 0.00072664097748 0.00109035601425\n",
      "231 40 -0.0339466258883 0.000744057579874 0.00109031344185\n",
      "231 50 -0.0414500273764 0.00074397993566 0.00109005446129\n",
      "valid_acc 98.375\n",
      "232 0 -0.0634429678321 0.000741430087522 0.00108981760216\n",
      "232 10 -0.0480462945998 0.000777990660475 0.00108920100201\n",
      "232 20 -0.0593526512384 0.000728687532661 0.00108952883408\n",
      "232 30 -0.0439423508942 0.00072228395894 0.00108897338104\n",
      "232 40 -0.0464712455869 0.000734841919261 0.00108827859462\n",
      "232 50 -0.0636687576771 0.000742015150073 0.00108797947037\n",
      "valid_acc 98.41333333333333\n",
      "233 0 -0.0472339242697 0.000775006302574 0.00108822719424\n",
      "233 10 -0.0433186031878 0.000731393042916 0.00108732107863\n",
      "233 20 -0.0504835359752 0.000736398962871 0.00108665782035\n",
      "233 30 -0.0476487614214 0.000726851200264 0.00108651541166\n",
      "233 40 -0.0302079003304 0.000705067795819 0.00108571302187\n",
      "233 50 -0.0499394722283 0.000743030514676 0.00108565926226\n",
      "valid_acc 98.43333333333332\n",
      "234 0 -0.051200542599 0.000724121928285 0.00108568325676\n",
      "234 10 -0.0314697176218 0.000722256094316 0.00108494485058\n",
      "234 20 -0.0473480448127 0.000737493019708 0.00108461920566\n",
      "234 30 -0.0469308495522 0.000703508779496 0.00108396799164\n",
      "234 40 -0.0420580990613 0.000736372442616 0.00108363815093\n",
      "234 50 -0.0417136289179 0.000735925007131 0.00108293061901\n",
      "valid_acc 98.43166666666666\n",
      "235 0 -0.0384425111115 0.000721478239446 0.00108226551871\n",
      "235 10 -0.0573138184845 0.000748230343284 0.00108232557507\n",
      "235 20 -0.0448119007051 0.000748798570755 0.0010829671918\n",
      "235 30 -0.0518580675125 0.000737455728307 0.00108334372601\n",
      "235 40 -0.0613280497491 0.000719913093002 0.00108354964271\n",
      "235 50 -0.0770723894238 0.000730136965015 0.00108279899422\n",
      "valid_acc 98.42166666666667\n",
      "236 0 -0.0761702805758 0.000766462617226 0.00108236341961\n",
      "236 10 -0.0507903657854 0.000716282259395 0.00108107082446\n",
      "236 20 -0.0782979205251 0.000745782276349 0.0010807175259\n",
      "236 30 -0.0482357218862 0.000716956305553 0.00107931555144\n",
      "236 40 -0.0517282932997 0.000716673271895 0.00107914220001\n",
      "236 50 -0.0554807037115 0.000722955959016 0.00107917003124\n",
      "valid_acc 98.42833333333333\n",
      "237 0 -0.0732577815652 0.000725651559802 0.00107880876527\n",
      "237 10 -0.0635118037462 0.000692978284044 0.00107802101712\n",
      "237 20 -0.0598705485463 0.000714107323859 0.00107840743737\n",
      "237 30 -0.0732093304396 0.000714709587819 0.00107840208223\n",
      "237 40 -0.0727673172951 0.000715342836861 0.00107767036674\n",
      "237 50 -0.049179520458 0.000733916875118 0.0010773121905\n",
      "valid_acc 98.43499999999999\n",
      "best valid_acc 98.43499999999999\n",
      "238 0 -0.0419949628413 0.000720376271982 0.0010769924279\n",
      "238 10 -0.0649216398597 0.000756669420449 0.00107599586735\n",
      "238 20 -0.067502990365 0.000761538053166 0.00107599160531\n",
      "238 30 -0.0606059134007 0.000715783575881 0.00107650888161\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "238 40 -0.0404366776347 0.000736800007925 0.00107593857851\n",
      "238 50 -0.0422367453575 0.000730835292335 0.00107542783597\n",
      "valid_acc 98.43833333333333\n",
      "best valid_acc 98.43833333333333\n",
      "239 0 -0.0471026450396 0.000726711764279 0.00107467749244\n",
      "239 10 -0.0461262501776 0.000722845846587 0.00107478125676\n",
      "239 20 -0.0672420337796 0.000730854813352 0.0010740406916\n",
      "239 30 -0.0425000414252 0.000693283879092 0.0010737913064\n",
      "239 40 -0.0436086393893 0.000722382252289 0.00107330618744\n",
      "239 50 -0.0430294647813 0.000745815339338 0.00107279597928\n",
      "valid_acc 98.4\n",
      "240 0 -0.0435610041022 0.000714353293585 0.00107175311822\n",
      "240 10 -0.0484903417528 0.000717347666879 0.00107076304249\n",
      "240 20 -0.0864793881774 0.00076078621618 0.00107174503238\n",
      "240 30 -0.0588762648404 0.000744634732276 0.00107170051989\n",
      "240 40 -0.0667910948396 0.000723769275775 0.00107166405869\n",
      "240 50 -0.0651135742664 0.00070883444831 0.00107166265388\n",
      "valid_acc 98.42833333333333\n",
      "241 0 -0.0432455763221 0.00072040432116 0.00107121590565\n",
      "241 10 -0.0617080554366 0.000754924369604 0.0010701235304\n",
      "241 20 -0.0598648004234 0.000722720797357 0.00106965771985\n",
      "241 30 -0.0527330897748 0.000716930758872 0.0010698830567\n",
      "241 40 -0.0374614223838 0.00071812137621 0.00106939121849\n",
      "241 50 -0.0657548680902 0.000748373747633 0.00106906590328\n",
      "valid_acc 98.41666666666666\n",
      "242 0 -0.0596330091357 0.000733061562523 0.00106763887427\n",
      "242 10 -0.080892637372 0.00076671861515 0.00106610954209\n",
      "242 20 -0.0384303964674 0.000720898155961 0.00106519823225\n",
      "242 30 -0.0436999835074 0.000753167644134 0.00106471408636\n",
      "242 40 -0.0646029636264 0.000738639712506 0.00106562220533\n",
      "242 50 -0.0510055013001 0.000723912013696 0.00106499909909\n",
      "valid_acc 98.405\n",
      "243 0 -0.0329557545483 0.000709160626995 0.0010634673712\n",
      "243 10 -0.0428574383259 0.000717681830924 0.00106316136696\n",
      "243 20 -0.041273124516 0.00068794298239 0.00106253952988\n",
      "243 30 -0.0451250597835 0.000701398414509 0.00106171993779\n",
      "243 40 -0.061485465616 0.000714974842397 0.00106105599404\n",
      "243 50 -0.0357345603406 0.000712363495733 0.00106080936874\n",
      "valid_acc 98.42\n",
      "244 0 -0.0638526380062 0.000717753457461 0.00106016514511\n",
      "244 10 -0.0334291085601 0.000717333360953 0.00105979363092\n",
      "244 20 -0.0486078858376 0.00070971291474 0.00105937615129\n",
      "244 30 -0.0243111029267 0.000706877528742 0.00105905439332\n",
      "244 40 -0.0439167842269 0.00073055494369 0.00105866964175\n",
      "244 50 -0.0601619072258 0.000722827834199 0.001059120428\n",
      "valid_acc 98.44333333333334\n",
      "best valid_acc 98.44333333333334\n",
      "245 0 -0.0458662211895 0.000705350969344 0.00105883240168\n",
      "245 10 -0.0373425148427 0.000720959072123 0.00105867347465\n",
      "245 20 -0.0340591520071 0.000737122581751 0.00105898141322\n",
      "245 30 -0.0445691272616 0.000711412579133 0.00105780038323\n",
      "245 40 -0.0475545935333 0.000731005401337 0.00105820772921\n",
      "245 50 -0.057092115283 0.00072168136891 0.0010580798308\n",
      "valid_acc 98.41166666666666\n",
      "246 0 -0.0462033562362 0.000715661413725 0.00105781303683\n",
      "246 10 -0.0652857795358 0.000721435437222 0.00105719702138\n",
      "246 20 -0.0600479468703 0.000736152911782 0.00105696880714\n",
      "246 30 -0.0417059846222 0.000724558914183 0.00105674393551\n",
      "246 40 -0.0492502376437 0.000726659089428 0.00105593181599\n",
      "246 50 -0.0434093549848 0.000742407694904 0.00105570772059\n",
      "valid_acc 98.42\n",
      "247 0 -0.0448309071362 0.000711165570763 0.00105556785142\n",
      "247 10 -0.0525945052505 0.000698679116721 0.00105637698606\n",
      "247 20 -0.0551238022745 0.000756717030675 0.00105661733577\n",
      "247 30 -0.0492401272058 0.000702550451482 0.00105569264627\n",
      "247 40 -0.0480381920934 0.000709526881164 0.00105625501615\n",
      "247 50 -0.0633437857032 0.000725376859338 0.00105632475247\n",
      "valid_acc 98.43166666666666\n",
      "248 0 -0.0425701625645 0.000775873165822 0.00105567213269\n",
      "248 10 -0.0376820489764 0.000734253082245 0.00105469242711\n",
      "248 20 -0.0410680621862 0.000736758622467 0.00105494903214\n",
      "248 30 -0.0529600791633 0.000777371477681 0.00105463401367\n",
      "248 40 -0.0399911627173 0.000700406024477 0.00105475395912\n",
      "248 50 -0.0457464382052 0.000756861825473 0.00105521109567\n",
      "valid_acc 98.41833333333334\n",
      "249 0 -0.0308221317828 0.000749161519786 0.00105405009057\n",
      "249 10 -0.0379980579019 0.000729859288904 0.00105345527493\n",
      "249 20 -0.0629379674792 0.000733539721379 0.0010534642952\n",
      "249 30 -0.047852806747 0.000738022007294 0.00105336652323\n",
      "249 40 -0.0559568032622 0.000719882998888 0.00105343302697\n",
      "249 50 -0.0446618236601 0.000750466443607 0.00105274009637\n",
      "valid_acc 98.42666666666666\n",
      "250 0 -0.043635122478 0.000688052392783 0.00105270364604\n",
      "250 10 -0.0583987496793 0.000768506870693 0.0010532949718\n",
      "250 20 -0.0440122969449 0.000739617281327 0.00105305336477\n",
      "250 30 -0.0255456790328 0.00073848621776 0.00105291415311\n",
      "250 40 -0.0624394416809 0.000717543176398 0.00105292298477\n",
      "250 50 -0.0387649722397 0.000747195687887 0.00105298142569\n",
      "valid_acc 98.43666666666667\n",
      "251 0 -0.0495283342898 0.000732173778495 0.0010528802567\n",
      "251 10 -0.0430175252259 0.000710145716398 0.00105263923992\n",
      "251 20 -0.0413231253624 0.000737349560713 0.00105246859197\n",
      "251 30 -0.0631574168801 0.000695563796294 0.00105246723114\n",
      "251 40 -0.0396432876587 0.000740673093346 0.00105189349824\n",
      "251 50 -0.0473195426166 0.000747818229405 0.0010514453104\n",
      "valid_acc 98.43333333333332\n",
      "252 0 -0.0290349088609 0.000734789969899 0.00105148201871\n",
      "252 10 -0.0617532469332 0.000721879333443 0.00105129129147\n",
      "252 20 -0.0333962850273 0.000712557737312 0.001051642738\n",
      "252 30 -0.0364178270102 0.000732392708891 0.00105031277826\n",
      "252 40 -0.0567881576717 0.000738094358215 0.00104948767378\n",
      "252 50 -0.0616801194847 0.000773434733591 0.00104846290457\n",
      "valid_acc 98.44833333333334\n",
      "best valid_acc 98.44833333333334\n",
      "253 0 -0.0420758239925 0.000741304323107 0.00104892208523\n",
      "253 10 -0.0357096083462 0.000725529011696 0.00104813051791\n",
      "253 20 -0.0551472567022 0.000711364630582 0.00104732622927\n",
      "253 30 -0.0489547848701 0.000778014655452 0.00104743701453\n",
      "253 40 -0.0492023229599 0.000729094155727 0.00104654138275\n",
      "253 50 -0.049294680357 0.000769789741116 0.00104649096327\n",
      "valid_acc 98.45\n",
      "best valid_acc 98.45\n",
      "254 0 -0.0647092834115 0.000718393056729 0.00104552714607\n",
      "254 10 -0.0408861041069 0.000749010601428 0.00104555853572\n",
      "254 20 -0.0371011942625 0.000758833064467 0.00104519334315\n",
      "254 30 -0.0521018430591 0.000772214389252 0.00104516171614\n",
      "254 40 -0.0430004447699 0.000753617066521 0.00104485404515\n",
      "254 50 -0.0592818967998 0.000746663636642 0.00104569276113\n",
      "valid_acc 98.43166666666666\n",
      "255 0 -0.0391067042947 0.000768248735156 0.00104485326181\n",
      "255 10 -0.0854958891869 0.00075226804002 0.00104501101655\n",
      "255 20 -0.0572206713259 0.000776581252128 0.00104465571401\n",
      "255 30 -0.0456946119666 0.000760025545229 0.00104309509415\n",
      "255 40 -0.0380666516721 0.000760445531545 0.00104256827338\n",
      "255 50 -0.0414435341954 0.000789590741353 0.00104275709364\n",
      "valid_acc 98.455\n",
      "best valid_acc 98.455\n",
      "256 0 -0.0482233464718 0.000752545367925 0.00104307775365\n",
      "256 10 -0.0423187725246 0.000716101617876 0.00104227858356\n",
      "256 20 -0.0489676333964 0.000759810799313 0.0010420830767\n",
      "256 30 -0.057039860636 0.000758473846883 0.00104115029408\n",
      "256 40 -0.0423108302057 0.000749319008427 0.00104102400559\n",
      "256 50 -0.0295582655817 0.000787205955195 0.00104028199979\n",
      "valid_acc 98.44000000000001\n",
      "257 0 -0.046095173806 0.000748879219443 0.00103999026305\n",
      "257 10 -0.0498191639781 0.000733512168079 0.00104044736211\n",
      "257 20 -0.0483681559563 0.00076668019129 0.00103988016036\n",
      "257 30 -0.0573412589729 0.000729175996409 0.0010392016102\n",
      "257 40 -0.0392733477056 0.000762251593072 0.00103806755755\n",
      "257 50 -0.0651442557573 0.000740963199841 0.00103769713182\n",
      "valid_acc 98.455\n",
      "best valid_acc 98.455\n",
      "258 0 -0.0591080933809 0.000741526858144 0.00103791289458\n",
      "258 10 -0.0513682365417 0.000781270874515 0.00103808661836\n",
      "258 20 -0.042244117707 0.000739326697482 0.00103809160164\n",
      "258 30 -0.0270368736237 0.000775078112758 0.00103724725194\n",
      "258 40 -0.0471573881805 0.000718485583465 0.00103731552832\n",
      "258 50 -0.0381532981992 0.000736511189327 0.00103699878657\n",
      "valid_acc 98.45333333333333\n",
      "259 0 -0.0375639274716 0.000733278191883 0.00103711867798\n",
      "259 10 -0.0235104579479 0.00075054977443 0.00103736217175\n",
      "259 20 -0.0529659204185 0.000751072348978 0.00103775018528\n",
      "259 30 -0.0399264991283 0.000762219981729 0.0010382263189\n",
      "259 40 -0.0475068837404 0.000741019030454 0.00103810589801\n",
      "259 50 -0.0423272140324 0.000733342768716 0.00103809350552\n",
      "valid_acc 98.455\n",
      "best valid_acc 98.455\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "260 0 -0.0457518994808 0.000760560552252 0.00103728428732\n",
      "260 10 -0.0462750792503 0.000750703803292 0.00103778002138\n",
      "260 20 -0.0445556901395 0.000744839392877 0.00103724652193\n",
      "260 30 -0.0480827204883 0.000751531764181 0.0010379596667\n",
      "260 40 -0.0507237054408 0.000746387551102 0.00103770546624\n",
      "260 50 -0.0334367491305 0.000746515677197 0.00103708644417\n",
      "valid_acc 98.44000000000001\n",
      "261 0 -0.0489831827581 0.000757318603818 0.00103700114727\n",
      "261 10 -0.0682558938861 0.000762739962495 0.00103686947146\n",
      "261 20 -0.039802968502 0.000755727595952 0.00103702842389\n",
      "261 30 -0.0625465065241 0.000772176900046 0.00103617715986\n",
      "261 40 -0.0688991621137 0.000778079223615 0.00103584864121\n",
      "261 50 -0.0517372898757 0.000742311144476 0.0010353854101\n",
      "valid_acc 98.42833333333333\n",
      "262 0 -0.0418926812708 0.000740557005886 0.00103468096816\n",
      "262 10 -0.0539679601789 0.000741323992463 0.00103348365963\n",
      "262 20 -0.0190537795424 0.000757216486271 0.00103335562787\n",
      "262 30 -0.0557400844991 0.00076307458248 0.00103288808268\n",
      "262 40 -0.0673588961363 0.000768494583559 0.00103245989418\n",
      "262 50 -0.045354090631 0.000755762133672 0.00103212562746\n",
      "valid_acc 98.42333333333333\n",
      "263 0 -0.0449746847153 0.000738847778403 0.00103199456447\n",
      "263 10 -0.0497609041631 0.000753055328094 0.00103148125735\n",
      "263 20 -0.0533204525709 0.000772714782586 0.00103139856126\n",
      "263 30 -0.0455002449453 0.000709812372121 0.00103099037207\n",
      "263 40 -0.0395807176828 0.000782851390184 0.00103134143095\n",
      "263 50 -0.0474548116326 0.000756959808592 0.00103102447492\n",
      "valid_acc 98.43166666666666\n",
      "264 0 -0.0413037352264 0.000781172371747 0.00103076133972\n",
      "264 10 -0.0409549362957 0.000742803161942 0.00103052062528\n",
      "264 20 -0.0402264781296 0.000757274522269 0.00103030466393\n",
      "264 30 -0.0473902560771 0.000724563884922 0.00103065640556\n",
      "264 40 -0.0569106154144 0.000733121176166 0.00103053140401\n",
      "264 50 -0.0515726804733 0.000741469799505 0.00102984040358\n",
      "valid_acc 98.44166666666668\n",
      "265 0 -0.0346588939428 0.000730634172335 0.00102865239507\n",
      "265 10 -0.0710377246141 0.000727278470804 0.00102784570281\n",
      "265 20 -0.0469171665609 0.000779910133152 0.00102780232217\n",
      "265 30 -0.0507390089333 0.000737185262289 0.00102791289027\n",
      "265 40 -0.0364461354911 0.000739540258332 0.00102801201241\n",
      "265 50 -0.0634424462914 0.0007348034858 0.00102773526094\n",
      "valid_acc 98.43166666666666\n",
      "266 0 -0.0447925329208 0.000740485502906 0.00102747978509\n",
      "266 10 -0.0381395295262 0.000775252400479 0.00102675523462\n",
      "266 20 -0.062223855406 0.00071949070082 0.00102673825755\n",
      "266 30 -0.0531221851707 0.000773820061133 0.00102620317932\n",
      "266 40 -0.0409418083727 0.000719401203629 0.00102590886216\n",
      "266 50 -0.0422615520656 0.000747073717426 0.00102556552495\n",
      "valid_acc 98.455\n",
      "best valid_acc 98.455\n",
      "267 0 -0.0337676815689 0.000725668027623 0.00102501057105\n",
      "267 10 -0.0480612367392 0.000747566803986 0.00102575523638\n",
      "267 20 -0.0488763861358 0.000752164221573 0.00102547579921\n",
      "267 30 -0.0294189359993 0.0007672708289 0.00102539010122\n",
      "267 40 -0.0496172867715 0.000734983693533 0.00102563478076\n",
      "267 50 -0.0625252053142 0.000715969488612 0.00102599055275\n",
      "valid_acc 98.44833333333334\n",
      "268 0 -0.0579074621201 0.000751313132066 0.00102625670738\n",
      "268 10 -0.0697681382298 0.000744763113227 0.00102732929858\n",
      "268 20 -0.0403208881617 0.000733908320396 0.0010267763603\n",
      "268 30 -0.023074278608 0.000739211004097 0.00102643194436\n",
      "268 40 -0.0504035390913 0.000758045921453 0.00102658280692\n",
      "268 50 -0.0537093505263 0.000722600514513 0.00102662903017\n",
      "valid_acc 98.47666666666667\n",
      "best valid_acc 98.47666666666667\n",
      "269 0 -0.0374209322035 0.00072563369405 0.00102689556973\n",
      "269 10 -0.0396690182388 0.0007417834737 0.00102593458196\n",
      "269 20 -0.0333807170391 0.000765393369114 0.00102555230863\n",
      "269 30 -0.0343588963151 0.000755308269427 0.00102580604013\n",
      "269 40 -0.0584964230657 0.000761917696717 0.00102530222288\n",
      "269 50 -0.0592219606042 0.000743032888393 0.00102602294555\n",
      "valid_acc 98.455\n",
      "270 0 -0.0629172846675 0.000722028426727 0.00102574531325\n",
      "270 10 -0.0403426513076 0.000767606239183 0.00102590762472\n",
      "270 20 -0.0526979714632 0.000719502539949 0.00102506850485\n",
      "270 30 -0.0721899420023 0.000752800451613 0.00102516498364\n",
      "270 40 -0.028428081423 0.000766139253614 0.00102496631259\n",
      "270 50 -0.0584368929267 0.00071586023096 0.00102568366999\n",
      "valid_acc 98.465\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 9*args.epochs + 1):\n",
    "\n",
    "  # train loop\n",
    "  model.eval()\n",
    "  for batch_idx, (data, target) in enumerate(train_loader):\n",
    "    if args.cuda:\n",
    "      data, target = data.cuda(), target.cuda()\n",
    "    data, target = Variable(data), Variable(target)\n",
    "    \n",
    "    solutions = es.ask()\n",
    "    reward = np.zeros(es.popsize)\n",
    "    \n",
    "    for i in range(es.popsize):\n",
    "      update_model(solutions[i], model, model_shapes)\n",
    "      output = model(data)\n",
    "      loss = F.nll_loss(output, target)\n",
    "      reward[i] = - loss.data[0]\n",
    "\n",
    "    best_raw_reward = reward.max()\n",
    "\n",
    "    es.tell(reward)\n",
    "\n",
    "    result = es.result()\n",
    "    \n",
    "    if (batch_idx % 10 == 0):\n",
    "      print(epoch, batch_idx, best_raw_reward, result[0].mean(), es.rms_stdev())\n",
    "\n",
    "  curr_solution = es.current_param()\n",
    "  update_model(curr_solution, model, model_shapes)\n",
    "\n",
    "  valid_acc = evaluate(model, valid_loader, print_mode=False)\n",
    "  training_log.append([epoch, valid_acc])\n",
    "  print('valid_acc', valid_acc * 100.)\n",
    "  if valid_acc >= best_valid_acc:\n",
    "    best_valid_acc = valid_acc\n",
    "    best_model = copy.deepcopy(model)\n",
    "    print('best valid_acc', best_valid_acc * 100.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59086/60000 (98.4767%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9847666666666667"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0612, Accuracy: 9798/10000 (97.9800%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9798"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59086/60000 (98.4767%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9847666666666667"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(best_model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.best_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59079/60000 (98.4650%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98465"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0612, Accuracy: 9798/10000 (97.9800%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9798"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59079/60000 (98.4650%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98465"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "update_model(es.current_param(), model, model_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59079/60000 (98.4650%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98465"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, valid_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0612, Accuracy: 9798/10000 (97.9800%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9798"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, test_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0498, Accuracy: 59079/60000 (98.4650%)\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.98465"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, train_loader, print_mode=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Average loss: 0.0612, Accuracy: 9798/10000 (97.9800%)\n",
      "\n",
      "final test acc 97.98\n"
     ]
    }
   ],
   "source": [
    "eval_acc = evaluate(best_model, test_loader)\n",
    "print('final test acc', eval_acc * 100.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 1, 5, 5])\n",
      "torch.Size([8])\n",
      "torch.Size([16, 8, 5, 5])\n",
      "torch.Size([16])\n",
      "torch.Size([10, 784])\n",
      "torch.Size([10])\n",
      "11274\n"
     ]
    }
   ],
   "source": [
    "param_count = 0\n",
    "for param in model.parameters():\n",
    "  print(param.data.shape)\n",
    "  param_count += np.product(param.data.shape)\n",
    "print(param_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params = []\n",
    "for param in orig_model.parameters():\n",
    "  orig_params.append(param.data.cpu().numpy().flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "orig_params_flat = np.concatenate(orig_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEexJREFUeJzt3X+IZWd9x/H3xySmRS1NmnFdN2snwrZlU2qUaZQqRWtr\nYvxjI5SwodiFpqxCFAX9Y6N/aJGFtFSFQpWuGtwWNV0wksWkLckiiKiJE4lJNjFmNRuyyya7/qr2\nn7RZv/1jTvQ6zsw9d+69OzPPvF9wmXOf8zznfOfM3c+cee65Z1NVSJLa9by1LkCSNF0GvSQ1zqCX\npMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalx5691AQCXXHJJzc7OrnUZkrSh3HfffT+oqplh\n/dZF0M/OzjI/P7/WZUjShpLkiT79nLqRpMYZ9JLUOINekho3NOiT/EaSe5N8O8nRJH/XtV+c5K4k\nj3VfLxoYc1OSY0keTXLVNL8BSdLK+pzRPwP8WVW9ArgCuDrJa4B9wJGq2gEc6Z6TZCewG7gcuBr4\neJLzplG8JGm4oUFfC/6ne3pB9yhgF3Cwaz8IXNst7wJurapnqupx4Bhw5USrliT11muOPsl5Se4H\nTgN3VdU9wJaqOtV1eQrY0i1vA54cGH6ia5MkrYFeQV9VZ6vqCuBS4Mokf7hofbFwlt9bkr1J5pPM\nnzlzZpShkqQRjHTVTVX9BPgyC3PvTyfZCtB9Pd11OwlsHxh2ade2eFsHqmququZmZoZ+sEuStEp9\nrrqZSfLb3fJvAn8BfAc4DOzpuu0Bbu+WDwO7k1yY5DJgB3DvpAuXWjG77w5m992x1mWoYX1ugbAV\nONhdOfM84FBVfSnJ14FDSW4AngCuA6iqo0kOAQ8DzwI3VtXZ6ZQvSRpmaNBX1QPAK5do/yHwxmXG\n7Af2j12dJGlsfjJWOkecotFaMeglqXEGvTYNz6i1WRn0ktQ4g16bzrk8q/cvCK0HBr02JadxtJkY\n9GqG4S0tzaCXpMYZ9JLUOINekhpn0GtTmNTcve8BaCMy6LXhDHvT1TCWfpVBL3X6/oLw6h5tNAa9\nJDXOoJekxvX5j0ckjcipHa0nntFLa8hfCDoXPKOXejCQtZF5Ri8twWBXSwx6SWqcQS9JjXOOXpoy\np4G01jyjVxMMU2l5Br20jNXe6mDYGH8p6Vwz6CWpcQa9NjXPrrUZDA36JNuTfDnJw0mOJnl31/6h\nJCeT3N89rhkYc1OSY0keTXLVNL8BaSneYVL6pT5X3TwLvLeqvpXkRcB9Se7q1n2sqv5xsHOSncBu\n4HLgpcDdSX6vqs5OsnBpPZnddwfHb37LSP2lc2XoGX1Vnaqqb3XLPwMeAbatMGQXcGtVPVNVjwPH\ngCsnUawkaXQjzdEnmQVeCdzTNb0ryQNJbklyUde2DXhyYNgJlvjFkGRvkvkk82fOnBm5cGmteVau\njaJ30Cd5IfAF4D1V9VPgE8DLgSuAU8BHRtlxVR2oqrmqmpuZmRllqCRpBL2CPskFLIT8Z6vqNoCq\nerqqzlbVz4FP8svpmZPA9oHhl3ZtkqQ1MPTN2CQBPg08UlUfHWjfWlWnuqdvBR7qlg8Dn0vyURbe\njN0B3DvRqqUpcTpGLepz1c1rgbcBDya5v2t7P3B9kiuAAo4DbweoqqNJDgEPs3DFzo1ecSNJa2do\n0FfVV4EsserOFcbsB/aPUZe0IUzyL4BRL9GU+vLulWraJILY6RxtdN4CQZIaZ9Brw1pvZ9redkHr\nlVM32tAMVmk4z+glqXEGvSQ1zqkbbXpO/6h1ntFLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0\nktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9J\njRsa9Em2J/lykoeTHE3y7q794iR3JXms+3rRwJibkhxL8miSq6b5DUiSVtbnjP5Z4L1VtRN4DXBj\nkp3APuBIVe0AjnTP6dbtBi4HrgY+nuS8aRQvSRpuaNBX1amq+la3/DPgEWAbsAs42HU7CFzbLe8C\nbq2qZ6rqceAYcOWkC5ck9TPSHH2SWeCVwD3Alqo61a16CtjSLW8DnhwYdqJrW7ytvUnmk8yfOXNm\nxLIlSX31DvokLwS+ALynqn46uK6qCqhRdlxVB6pqrqrmZmZmRhkqSRpBr6BPcgELIf/Zqrqta346\nydZu/VbgdNd+Etg+MPzSrk2StAb6XHUT4NPAI1X10YFVh4E93fIe4PaB9t1JLkxyGbADuHdyJUuS\nRnF+jz6vBd4GPJjk/q7t/cDNwKEkNwBPANcBVNXRJIeAh1m4YufGqjo78colSb0MDfqq+iqQZVa/\ncZkx+4H9Y9QlSZoQPxkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+gl\nqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIa\nZ9BLUuMMeklqnEEvSY0z6CWpcUODPsktSU4neWig7UNJTia5v3tcM7DupiTHkjya5KppFS5J6qfP\nGf1ngKuXaP9YVV3RPe4ESLIT2A1c3o35eJLzJlWsJGl0Q4O+qr4C/Kjn9nYBt1bVM1X1OHAMuHKM\n+iRJYxpnjv5dSR7opnYu6tq2AU8O9DnRtf2aJHuTzCeZP3PmzBhlSJJWstqg/wTwcuAK4BTwkVE3\nUFUHqmququZmZmZWWYYkaZhVBX1VPV1VZ6vq58An+eX0zElg+0DXS7s2SdIaWVXQJ9k68PStwHNX\n5BwGdie5MMllwA7g3vFKlCSN4/xhHZJ8Hng9cEmSE8AHgdcnuQIo4DjwdoCqOprkEPAw8CxwY1Wd\nnU7pkqQ+hgZ9VV2/RPOnV+i/H9g/TlGSpMnxk7GS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9\nJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS\n4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1LihQZ/kliSnkzw00HZxkruSPNZ9vWhg3U1JjiV5\nNMlV0ypcktRPnzP6zwBXL2rbBxypqh3Ake45SXYCu4HLuzEfT3LexKqVJI1saNBX1VeAHy1q3gUc\n7JYPAtcOtN9aVc9U1ePAMeDKCdUqSVqF1c7Rb6mqU93yU8CWbnkb8ORAvxNdmyRpjYz9ZmxVFVCj\njkuyN8l8kvkzZ86MW4YkaRmrDfqnk2wF6L6e7tpPAtsH+l3atf2aqjpQVXNVNTczM7PKMiRJw6w2\n6A8De7rlPcDtA+27k1yY5DJgB3DveCVKksZx/rAOST4PvB64JMkJ4IPAzcChJDcATwDXAVTV0SSH\ngIeBZ4Ebq+rslGqXJPUwNOir6vplVr1xmf77gf3jFCVJmhw/GStJjTPoJalxBr0kNc6gl6TGGfSS\n1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mN\nM+glqXEGvSQ1zqDXOTe77461LkHaVAx6SWqcQS81wL+StBKDXhM3u+8Og0daRwx6SWqcQa8NYbP9\nlTCN7/W5Y7iZjqMWGPTaUAwpaXTnjzM4yXHgZ8BZ4NmqmktyMfDvwCxwHLiuqn48XpmSRvHcL8Tj\nN79ljSvRejCJM/o3VNUVVTXXPd8HHKmqHcCR7rkkaY1MY+pmF3CwWz4IXDuFfUiSehpr6gYo4O4k\nZ4F/qaoDwJaqOtWtfwrYMuY+JPWw3HSN72to3KB/XVWdTPJi4K4k3xlcWVWVpJYamGQvsBfgZS97\n2ZhlSFqKIS8Yc+qmqk52X08DXwSuBJ5OshWg+3p6mbEHqmququZmZmbGKUPrlCEzGR5HjWvVQZ/k\nBUle9Nwy8CbgIeAwsKfrtge4fdwiJUmrN87UzRbgi0me287nquo/k3wTOJTkBuAJ4Lrxy9RG1Wfe\neLlLADfjJYKDx8UzeU3KqoO+qr4PvGKJ9h8CbxynKG0umzHQpXPJT8Zq3fAMVpoOg17aYIbdr8Zf\nmFps3MsrpV8YFjAG0GTN7rvD6S71YtBLG5i/PNWHUzfSBmKwazUMep0TowaUgSZNjlM3WlcM+KV5\nXDSOVC15K5pzam5urubn59e6DI3BINqYfDN3Y0ty38At4pfl1I0kNc6gl6TGGfSS1DiDXpIaZ9BL\nUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1\nzqCXpMZNLeiTXJ3k0STHkuyb1n4kSSubStAnOQ/4Z+DNwE7g+iQ7p7EvSdoI1vL/VZ7WGf2VwLGq\n+n5V/S9wK7BrSvv6tQM4u++OJdv6jO2zr6W2v9T2BvuuZv8rjetTSx/j7L9vH61fi1+vK/Vbav1q\nf/4r/Xvou70+/74Wb3OUvkvtZ9z+a+X8KW13G/DkwPMTwKuntC9gvN+WS409fvNbfqX9+M1vGXuf\ny21v2H4G+4yz/rl1g8uL97/autWGwdfRUq+pxa+dpdpHfa31fW0vrqtP7X0s9z2N0nel9pVqheW/\n70lKVU1+o8lfAldX1d92z98GvLqq3jnQZy+wt3v6+8Cjq9jVJcAPxix3WtZrbdY1uvVa23qtC9Zv\nba3V9btVNTOs07TO6E8C2weeX9q1/UJVHQAOjLOTJPNVNTfONqZlvdZmXaNbr7Wt17pg/da2Weua\n1hz9N4EdSS5L8nxgN3B4SvuSJK1gKmf0VfVskncC/wWcB9xSVUensS9J0sqmNXVDVd0J3Dmt7XfG\nmvqZsvVam3WNbr3Wtl7rgvVb26asaypvxkqS1g9vgSBJjVv3QZ/k4iR3JXms+3rREn22J/lykoeT\nHE3y7lHGT6uurt8tSU4neWhR+4eSnExyf/e4ZhJ1Tai2tT5mS94+Y9LHbNhtOrLgn7r1DyR5Vd+x\n4xqztuNJHuyO0fw5rusPknw9yTNJ3jfK2DWsa2rHq2dtf9X9DB9M8rUkr+g7treqWtcP4B+Afd3y\nPuDvl+izFXhVt/wi4LvAzr7jp1VXt+5PgVcBDy1q/xDwvrU6ZkNqW7NjxsKb998DXg48H/j2wM9y\nYsdspf0M9LkG+A8gwGuAe/qOXavaunXHgUum8LrqU9eLgT8G9g/+rKZ5zMapa5rHa4Ta/gS4qFt+\n8zReZ+v+jJ6FWycc7JYPAtcu7lBVp6rqW93yz4BHWPh0bq/x06qrq+crwI8mtM++xq1tLY/Zubp9\nRp/97AL+tRZ8A/jtJFvPQY3j1DZNQ+uqqtNV9U3g/0Ydu0Z1TVuf2r5WVT/unn6Dhc8d9Rrb10YI\n+i1VdapbfgrYslLnJLPAK4F7VjN+WnUt413dn2y3TGp6ZEK1reUxW+r2GdsGnk/qmA3bz0p9+owd\nxzi1ARRwd5L7svAJ9HNZ1zTGTnvb0zpeMHptN7Dwl9pqxi5rapdXjiLJ3cBLllj1gcEnVVVJlr1M\nKMkLgS8A76mqny5eP2z8tOpaxieAD7PwIvsw8BHgb9ZJbasev56P2Sbyuqo6meTFwF1JvtP99aal\nrYvjleQNLAT96ya97XUR9FX158utS/J0kq1Vdar70/T0Mv0uYCHkP1tVtw2s6jV+WnWtsO2nB7b1\nSeBLI46fWm2s7TFb9vYZ4x6zvvvp0eeCHmPHMU5tVNVzX08n+SILUwCTCK4+dU1j7FS3PcXj1bu2\nJH8EfAp4c1X9cJSxfWyEqZvDwJ5ueQ9w++IOSQJ8Gnikqj466vhp1bWSRfOpbwUeWq7vKoz7Pa/l\nMVv29hkTPmZ9btNxGPjr7gqX1wD/3U09TfsWH6uuLckLkrwIIMkLgDcxudfWON/3NI/Zqrc95ePV\nq7YkLwNuA95WVd8dZWxv03ineZIP4HeAI8BjwN3AxV37S4E7u+XXsfDn/APA/d3jmpXGn4u6uuef\nB06x8CbQCeCGrv3fgAe7mg8DW8/lMRtS21ofs2tYuHLqe8AHBtonesyW2g/wDuAd3XJY+A90vtft\nd25YjRP8Ga6qNhau0Ph29zg66dp61PWS7rX0U+An3fJvTfuYrbauaR+vnrV9Cvgxv8yu+Um/zvxk\nrCQ1biNM3UiSxmDQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuP8Hm+2DKUyf1DkAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4e3b6104a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(orig_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_params = []\n",
    "for param in best_model.parameters():\n",
    "  final_params.append(param.data.cpu().numpy().flatten())\n",
    "final_params_flat = np.concatenate(final_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEPpJREFUeJzt3X+s3Xddx/Hnyw6GQZTVXUppO+9IGrQz4Udu6gRCplNW\nwdiZ6FISsTE1DclQTEz0ThP5q8n0DyImzqQBtEZgNiiuoQPSVRZiAht3Mtjaba6wLW3TXwwB8Y/i\nxts/7rfjrOvtOaf3nHvu/dznI2nO5/v5fr7nfD799r7Op5/zPd+bqkKS1K4fm3QHJEnjZdBLUuMM\neklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGnfVpDsAcO2119b09PSkuyFJK8pDDz30raqa\n6tduWQT99PQ0c3Nzk+6GJK0oSZ4ZpJ1LN5LUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6g\nl6TGGfSS1DiDXlompmcPTroLapRBL0mNM+glqXEGvbSMTM8edAlHI2fQS1LjDHpJapxBL0mNM+gl\nqXEGvSQ1zqCXpMYZ9NIy5CWWGiWDXpIaN1DQJ3l1kk8leTzJY0l+McnaJIeSPNk9XtPT/o4kx5I8\nkeSW8XVfktTPoDP6DwOfq6qfBd4IPAbMAoerajNwuNsmyRZgB3ADsA24K8maUXdckjSYvkGf5KeA\ndwAfBaiqH1TVd4DtwL6u2T7g1q68Hbi7qs5X1VPAMWDrqDsuSRrMIDP664FzwN8n+WqSjyR5JbCu\nqk51bU4D67ryBuB4z/EnujpJ0gQMEvRXAW8B/q6q3gz8L90yzQVVVUAN88JJdieZSzJ37ty5YQ6V\nJA1hkKA/AZyoqge67U8xH/xnkqwH6B7PdvtPApt6jt/Y1b1IVe2tqpmqmpmamrrS/kuS+ugb9FV1\nGjie5A1d1c3AUeAAsLOr2wnc05UPADuSXJ3kemAz8OBIey1JGthVA7b7A+DjSV4OfBP4PebfJPYn\n2QU8A9wGUFVHkuxn/s3gOeD2qnp+5D2XJA1koKCvqoeBmUvsunmB9nuAPYvolyRpRPxmrCQ1zqCX\npMYNukYvaUy8gZnGzRm9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCX\npMYZ9JLUOINeWqamZw96HxyNhEEvLXOGvRbLoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+gl\nqXEDBX2Sp5M8kuThJHNd3dokh5I82T1e09P+jiTHkjyR5JZxdV6S1N8wM/pfqqo3VdVMtz0LHK6q\nzcDhbpskW4AdwA3ANuCuJGtG2GdJ0hAWs3SzHdjXlfcBt/bU311V56vqKeAYsHURryM1y2+9aikM\nGvQF3JfkoSS7u7p1VXWqK58G1nXlDcDxnmNPdHWSpAm4asB2b6+qk0leAxxK8njvzqqqJDXMC3dv\nGLsBrrvuumEOlSQNYaAZfVWd7B7PAp9mfinmTJL1AN3j2a75SWBTz+Ebu7qLn3NvVc1U1czU1NSV\nj0BaBbyTpRajb9AneWWSV10oA+8EHgUOADu7ZjuBe7ryAWBHkquTXA9sBh4cdcclSYMZZOlmHfDp\nJBfaf6KqPpfkK8D+JLuAZ4DbAKrqSJL9wFHgOeD2qnp+LL2XJPXVN+ir6pvAGy9R/yxw8wLH7AH2\nLLp3UqNchtFS8puxktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn\n0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXlpi/nYpLTWDXpIaZ9BLUuMMeklqnEEv\nSY0z6CWpcQMHfZI1Sb6a5DPd9tokh5I82T1e09P2jiTHkjyR5JZxdFySNJhhZvQfAB7r2Z4FDlfV\nZuBwt02SLcAO4AZgG3BXkjWj6a4kaVgDBX2SjcC7gY/0VG8H9nXlfcCtPfV3V9X5qnoKOAZsHU13\npdXNa/B1JQad0f818CfAD3vq1lXVqa58GljXlTcAx3vanejqXiTJ7iRzSebOnTs3XK8lSQPrG/RJ\nfh04W1UPLdSmqgqoYV64qvZW1UxVzUxNTQ1zqCRpCFcN0OZtwG8keRfwCuAnk/wTcCbJ+qo6lWQ9\ncLZrfxLY1HP8xq5OkjQBfWf0VXVHVW2sqmnmP2T996r6HeAAsLNrthO4pysfAHYkuTrJ9cBm4MGR\n91ySNJBBZvQLuRPYn2QX8AxwG0BVHUmyHzgKPAfcXlXPL7qnkoAffSD79J3vnnBPtFIMFfRVdT9w\nf1d+Frh5gXZ7gD2L7JskaQT8ZqwkNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9\nJDXOoJekxhn0ktS4xdzUTNIQ/O1QmhRn9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6g\nl6TGGfTSCuUXsDQog15awaZnDxr46sugl6TG9b3XTZJXAF8Eru7af6qqPphkLfDPwDTwNHBbVf13\nd8wdwC7geeAPq+rzY+m9tAI449akDTKjPw/8clW9EXgTsC3JjcAscLiqNgOHu22SbAF2ADcA24C7\nkqwZR+clSf31Dfqa9/1u82XdnwK2A/u6+n3ArV15O3B3VZ2vqqeAY8DWkfZakjSwgdbok6xJ8jBw\nFjhUVQ8A66rqVNfkNLCuK28AjvccfqKru/g5dyeZSzJ37ty5Kx6AJOnyBgr6qnq+qt4EbAS2Jvn5\ni/YX87P8gVXV3qqaqaqZqampYQ6VJA1hqKtuquo7wBeYX3s/k2Q9QPd4tmt2EtjUc9jGrk6SNAF9\ngz7JVJJXd+UfB34VeBw4AOzsmu0E7unKB4AdSa5Ocj2wGXhw1B2XJA1mkF8luB7Y110582PA/qr6\nTJIvAfuT7AKeAW4DqKojSfYDR4HngNur6vnxdF+S1E/foK+qrwNvvkT9s8DNCxyzB9iz6N5JkhbN\nb8ZKUuMMeklqnEEvSY0z6CWpcQa9NEbe0EzLgUEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQ\nS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6qQHe916XY9BLI2boarnp\nG/RJNiX5QpKjSY4k+UBXvzbJoSRPdo/X9BxzR5JjSZ5Icss4ByBp3vTsQd9kdEmDzOifA/64qrYA\nNwK3J9kCzAKHq2ozcLjbptu3A7gB2AbclWTNODovServqn4NquoUcKor/0+Sx4ANwHbgpq7ZPuB+\n4E+7+rur6jzwVJJjwFbgS6PuvLRcObPWcjLUGn2SaeDNwAPAuu5NAOA0sK4rbwCO9xx2oquTJE3A\nwEGf5CeAfwH+qKq+17uvqgqoYV44ye4kc0nmzp07N8yhkqQhDBT0SV7GfMh/vKr+tas+k2R9t389\ncLarPwls6jl8Y1f3IlW1t6pmqmpmamrqSvsvSepjkKtuAnwUeKyqPtSz6wCwsyvvBO7pqd+R5Ook\n1wObgQdH12VJ0jD6fhgLvA14L/BIkoe7uj8D7gT2J9kFPAPcBlBVR5LsB44yf8XO7VX1/Mh7Lkka\nyCBX3fwHkAV237zAMXuAPYvolyRpRPxmrCQ1zqCXpMYZ9JLUuEE+jJU0AL8Nq+XKGb0kNc6gl6TG\nGfSS1DiDXpIaZ9BLUuMMeqkxXv2jixn0ktQ4g16SGmfQS1LjDHqpQdOzB12r1wsMeklqnEEvSY0z\n6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1Lj/FWC0hXq/ULS03e+e4I9kS6v74w+yceSnE3yaE/d\n2iSHkjzZPV7Ts++OJMeSPJHklnF1XJI0mEGWbv4B2HZR3SxwuKo2A4e7bZJsAXYAN3TH3JVkzch6\nKy1T3m5Ay1nfoK+qLwLfvqh6O7CvK+8Dbu2pv7uqzlfVU8AxYOuI+irpCnjfG13ph7HrqupUVz4N\nrOvKG4DjPe1OdHWSJsCAF4zgqpuqKqCGPS7J7iRzSebOnTu32G5IkhZwpUF/Jsl6gO7xbFd/EtjU\n025jV/cSVbW3qmaqamZqauoKuyFJ6udKL688AOwE7uwe7+mp/0SSDwGvAzYDDy62k9JysZKXQqZn\nD3oZ6CrVN+iTfBK4Cbg2yQngg8wH/P4ku4BngNsAqupIkv3AUeA54Paqen5MfZc0pAtvVAb+6tI3\n6KvqPQvsunmB9nuAPYvplCRpdLwFgiQ1zqCXpMYZ9JLUOG9qJg1gJV9tIzmjl1Y5b5HQPoNekhpn\n0Et9tDjbdRa/urhGLy1gNQThahijnNFLUvMMeukSnOmqJQa9pJfwja4tBr0kwHBvmUEvSY0z6CWp\ncV5eKfVw+UItckYvdQx5tcqg16rnt0Qvz7+blc+lG0kvMNTb5Ixeq5rBptXAGb1WJQO+v4X+jvwF\n4yuPQa+m9YaSAXXlfGNc2Vy60apgUGk1M+glXTGvWFoZUlXjeeJkG/BhYA3wkaq6c6G2MzMzNTc3\nN5Z+qG2XWprRZLgktvSSPFRVM/3ajWVGn2QN8LfArwFbgPck2TKO15LApZnl5MIs/+Jz4jmanHF9\nGLsVOFZV3wRIcjewHTg6ptdTw3oDwpn78nWp8zI9e/BFM30/HJ+MsSzdJPktYFtV/X63/V7gF6rq\n/Zdqv9ilm4v/MY3bKP+BXhxi43LxD2G/1xr2B/RSP+QXh/KF9r3PbWhrqYw7I4bNhVHkyKBLNxML\n+iS7gd3d5huAJ4Z8mWuBb42guyuJY14dVtuYV9t4YXRj/pmqmurXaFxLNyeBTT3bG7u6F1TVXmDv\nlb5AkrlB3sla4phXh9U25tU2Xlj6MY/r8sqvAJuTXJ/k5cAO4MCYXkuSdBljmdFX1XNJ3g98nvnL\nKz9WVUfG8VqSpMsb2y0Qqupe4N5xPT+LWPZZwRzz6rDaxrzaxgtLPOaxfWFKkrQ8eAsESWrcign6\nJL+d5EiSHyZZ8NPqJE8neSTJw0lW9H0VhhjztiRPJDmWZHYp+zhqSdYmOZTkye7xmgXarejz3O+c\nZd7fdPu/nuQtk+jnKA0w5puSfLc7pw8n+YtJ9HNUknwsydkkjy6wf+nOcVWtiD/AzzF/vf39wMxl\n2j0NXDvp/i7VmJn/sPsbwOuBlwNfA7ZMuu+LGPNfAbNdeRb4y9bO8yDnDHgX8FkgwI3AA5Pu9xKM\n+SbgM5Pu6wjH/A7gLcCjC+xfsnO8Ymb0VfVYVQ37paoVbcAxv3C7iar6AXDhdhMr1XZgX1feB9w6\nwb6MyyDnbDvwjzXvy8Crk6xf6o6OUGv/Tvuqqi8C375MkyU7xysm6IdQwH1JHuq+fdu6DcDxnu0T\nXd1Kta6qTnXl08C6Bdqt5PM8yDlr7bwOOp63dssYn01yw9J0bWKW7Bwvq98wleQ+4LWX2PXnVXXP\ngE/z9qo6meQ1wKEkj3fvrMvSiMa8olxuzL0bVVVJFrosbEWdZw3kP4Hrqur7Sd4F/BuwecJ9asKy\nCvqq+pURPMfJ7vFskk8z/1/GZRsAIxhz39tNLDeXG3OSM0nWV9Wp7r+xZxd4jhV1ni8yyDlbcee1\nj0Fui/K9nvK9Se5Kcm1VtXofnCU7x00t3SR5ZZJXXSgD7wQu+Yl3Q1q73cQBYGdX3gm85H81DZzn\nQc7ZAeB3uyszbgS+27OktRL1HXOS1yZJV97KfD49u+Q9XTpLd44n/cn0EJ9g/ybza1jngTPA57v6\n1wH3duXXM/9p/teAI8wvf0y87+Mcc/3o0/v/Yv6qhpU+5p8GDgNPAvcBa1s8z5c6Z8D7gPd15TD/\ny3u+ATzCZa40Wyl/Bhjz+7vz+TXgy8BbJ93nRY73k8Ap4P+6n+NdkzrHfjNWkhrX1NKNJOmlDHpJ\napxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhr3/4AKiu8dYT9EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4e3b2827f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.hist(final_params_flat, bins=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
